Merge branch 'main' of https://github.com/davidekete/netplan into Fix…

…-docs-directory-structure-to-reflect-Diátaxis

include/netdef.h

@@ -54,15 +54,15 @@ netplan_netdef_get_filepath(const NetplanNetDefinition* netdef, char* out_buffer
 
 /**
  * @brief Get the specific @ref NetplanBackend defined for this @ref NetplanNetDefinition.
- * @param[in] np_state The @ref NetplanState to query
+ * @param[in] netdef   The @ref NetplanNetDefinition to query
  * @return             Enumeration value, specifiying the @ref NetplanBackend
  */
 NETPLAN_PUBLIC NetplanBackend
 netplan_netdef_get_backend(const NetplanNetDefinition* netdef);
 
 /**
  * @brief Get the interface type for a given @ref NetplanNetDefinition.
- * @param[in] np_state The @ref NetplanState to query
+ * @param[in] netdef   The @ref NetplanNetDefinition to query
  * @return             Enumeration value of @ref NetplanDefType, specifiying the interface type
  */
 NETPLAN_PUBLIC NetplanDefType

include/state.h

@@ -124,10 +124,10 @@ netplan_state_update_yaml_hierarchy(
 /**
  * @brief   Dump the whole @ref NetplanState into a single YAML file.
  * @details Ignoring the origin of each @ref NetplanNetDefinition.
- * @param[in]  np_state The @ref NetplanState for which to generate the configuration
- * @param[in]  out_fd   File descriptor to an opened file into which to dump the content
- * @param[out] error    Filled with a @ref NetplanError in case of failure
- * @return              Indication of success or failure
+ * @param[in]  np_state  The @ref NetplanState for which to generate the configuration
+ * @param[in]  output_fd File descriptor to an opened file into which to dump the content
+ * @param[out] error     Filled with a @ref NetplanError in case of failure
+ * @return               Indication of success or failure
  */
 NETPLAN_PUBLIC gboolean
 netplan_state_dump_yaml(

include/types.h

@@ -106,7 +106,7 @@ typedef struct _NetplanStateIterator NetplanStateIterator;
  * @note    The idea is based on the GLib implementation of iterators.
  */
 struct _NetplanStateIterator {
-    void* placeholder;
+    void* placeholder; ///< Just a placeholder in memory
 };
 
 /*

netplan_cli/cli/commands/apply.py

@@ -25,8 +25,6 @@
 import glob
 import subprocess
 import shutil
-import tempfile
-import filecmp
 import time
 
 from .. import utils
@@ -110,6 +108,7 @@ def command_apply(self, run_generate=True, sync=False, exit_on_error=True, state
                 return
 
         ovs_cleanup_service = '/run/systemd/system/netplan-ovs-cleanup.service'
+        old_files_networkd = bool(glob.glob('/run/systemd/network/*netplan-*'))
         old_ovs_glob = glob.glob('/run/systemd/system/netplan-ovs-*')
         # Ignore netplan-ovs-cleanup.service, as it can always be there
         if ovs_cleanup_service in old_ovs_glob:
@@ -119,39 +118,30 @@ def command_apply(self, run_generate=True, sync=False, exit_on_error=True, state
         nm_ifaces = utils.nm_interfaces(old_nm_glob, utils.get_interfaces())
         old_files_nm = bool(old_nm_glob)
 
-        restart_networkd = False
-        with tempfile.TemporaryDirectory() as tmp_dir:
-            # needs to be a subfolder as copytree wants to create it
-            run_systemd_network = '/run/systemd/network'
-            old_files_dir = os.path.join(tmp_dir, 'cfg')
-            has_old_networkd_config = os.path.isdir(run_systemd_network)
-            if has_old_networkd_config:
-                shutil.copytree(run_systemd_network, old_files_dir)
-
-            generator_call = []
-            generate_out = None
-            if 'NETPLAN_PROFILE' in os.environ:
-                generator_call.extend(['valgrind', '--leak-check=full'])
-                generate_out = subprocess.STDOUT
-
-            generator_call.append(utils.get_generator_path())
-            if run_generate and subprocess.call(generator_call, stderr=generate_out) != 0:
-                if exit_on_error:
-                    sys.exit(os.EX_CONFIG)
-                else:
-                    raise ConfigurationError("the configuration could not be generated")
+        generator_call = []
+        generate_out = None
+        if 'NETPLAN_PROFILE' in os.environ:
+            generator_call.extend(['valgrind', '--leak-check=full'])
+            generate_out = subprocess.STDOUT
 
-            # Restart networkd if something in the configuration changed
-            has_new_networkd_config = os.path.isdir(run_systemd_network)
-            if has_old_networkd_config != has_new_networkd_config:
-                restart_networkd = True
-            elif has_old_networkd_config and has_new_networkd_config:
-                comp = filecmp.dircmp(run_systemd_network, old_files_dir)
-                if comp.left_only or comp.right_only or comp.diff_files:
-                    restart_networkd = True
+        generator_call.append(utils.get_generator_path())
+        if run_generate and subprocess.call(generator_call, stderr=generate_out) != 0:
+            if exit_on_error:
+                sys.exit(os.EX_CONFIG)
+            else:
+                raise ConfigurationError("the configuration could not be generated")
 
         devices = utils.get_interfaces()
 
+        # Re-start service when
+        # 1. We have configuration files for it
+        # 2. Previously we had config files for it but not anymore
+        # Ideally we should compare the content of the *netplan-* files before and
+        # after generation to minimize the number of re-starts, but the conditions
+        # above works too.
+        restart_networkd = bool(glob.glob('/run/systemd/network/*netplan-*'))
+        if not restart_networkd and old_files_networkd:
+            restart_networkd = True
         restart_ovs_glob = glob.glob('/run/systemd/system/netplan-ovs-*')
         # Ignore netplan-ovs-cleanup.service, as it can always be there
         if ovs_cleanup_service in restart_ovs_glob:

netplan_cli/cli/core.py

@@ -35,7 +35,7 @@ def __init__(self):
                          description='Network configuration in YAML',
                          leaf=False)
         os.environ.update({
-            'LC_ALL': 'C',
+            'LC_ALL': 'C.UTF-8',
             'PATH': os.getenv('PATH', FALLBACK_PATH)})
 
     def parse_args(self):

netplan_cli/cli/state.py

@@ -28,6 +28,7 @@
 from io import StringIO
 from socket import AF_INET, AF_INET6, inet_ntop
 from typing import Dict, List, Type, Union
+from urllib import parse
 
 import yaml
 
@@ -59,6 +60,10 @@
     'vrf': 'vrf',
     'vxlan': 'tunnel',
 
+    # Used for wifi testing.
+    # It's the type of the interface hwsim0 created by the mac80211_hwsim driver
+    'ieee80211_radiotap': 'wifi',
+
     # Netplan netdef types
     'wifis': 'wifi',
     'ethernets': 'ethernet',
@@ -314,7 +319,23 @@ def netdef_id(self) -> str:
                 'run/NetworkManager/system-connections/netplan-')[1].split('.nmconnection')[0]
             if self.nm.get('type', '') == '802-11-wireless':
                 ssid = self.query_nm_ssid(self.nm.get('name'))
-                if ssid:  # XXX: escaping needed?
+                if ssid not in netdef:
+                    # If the plain SSID in not found in the netdef here
+                    # it's probably because it contains non-ascii characters that
+                    # were escaped in the file name. We need to do the same here to
+                    # be able to extract it from the file name.
+                    # In this case, Network Manager will save the SSID using the format "b1;b2;b3...;"
+                    # instead of a non-ascii string.
+                    # In src/nm.c we use g_uri_escape_string() to create the file name.
+
+                    # Transform the SSID to the same format used by Network Manager
+                    ssid_encoded = ssid.encode('utf-8')
+                    ssid_bytes = [str(b) for b in ssid_encoded]
+                    ssid_nm_escaped = ';'.join(ssid_bytes) + ';'
+
+                    # Escape characters in the same way we do in src/nm.c.
+                    ssid = parse.quote(ssid_nm_escaped)
+                if ssid:
                     netdef = netdef.split('-' + ssid)[0]
             return netdef
         return None
@@ -328,13 +349,22 @@ def vendor(self) -> str:
     @property
     def ssid(self) -> str:
         if self.type == 'wifi':
+            if self.backend == "NetworkManager":
+                return self.query_nm_ssid(self.nm.get('name', ''))
             # XXX: available from networkctl's JSON output as of v250:
             #      https://github.com/systemd/systemd/commit/da7c995
+            # TODO: Retrieving the SSID from systemd seems to not be reliable.
+            #       Sometimes it will return "(null)".
             for line in self._networkctl.splitlines():
                 line = line.strip()
-                key = 'WiFi access point: '
-                if line.startswith(key):
-                    ssid = line[len(key):-len(' (xB:SS:ID:xx:xx:xx)')].strip()
+                key = r'^Wi-?Fi access point: (.*) \(.*\)'
+                if match := re.match(key, line):
+                    ssid = match.group(1)
+                    # TODO: Find a better way to retrieve the SSID
+                    # networkctl will return a non-ascii SSID using the octal notation below:
+                    # '\\303\\241\\303\\251\\303\\255\\303\\263\\303...
+                    # Here we handle the escaping, the encoding of individual bytes and the final decoding to utf-8
+                    ssid = ssid.encode('latin1').decode('unicode-escape').encode('latin1').decode('utf-8')
                     return ssid if ssid else None
         return None
 

src/networkd.c

@@ -1341,6 +1341,8 @@ write_wpa_unit(const NetplanNetDefinition* def, const char* rootdir)
         g_string_append(s, " -Dnl80211,wext\n");
     }
 
+    g_string_append_printf(s, "ExecReload=/sbin/wpa_cli -i %s reconfigure\n", stdouth);
+
     g_autofree char* new_s = _netplan_scrub_systemd_unit_contents(s->str);
     g_string_free(s, TRUE);
     s = g_string_new(new_s);

src/parse.c

@@ -392,7 +392,7 @@ handle_special_macaddress_option(NetplanParser* npp, yaml_node_t* node, void* en
     g_assert(entryptr != NULL);
     g_assert(node->type == YAML_SCALAR_NODE);
 
-    if (!_is_macaddress_special_nm_option(scalar(node)) &&
+    if (!_is_macaddress_special_nm_option(npp->current.netdef, scalar(node)) &&
         !_is_macaddress_special_nd_option(scalar(node)))
         return FALSE;
 
@@ -695,7 +695,7 @@ handle_netdef_set_mac(NetplanParser* npp, yaml_node_t* node, const void* data, G
         if (!handle_special_macaddress_option(npp, node, npp->current.netdef, data, NULL)) {
             return yaml_error(npp, node, error,
                               "Invalid MAC address '%s', must be XX:XX:XX:XX:XX:XX, XX:XX:XX:XX:XX:XX:XX:XX:XX:XX:XX:XX:XX:XX:XX:XX:XX:XX:XX:XX"
-                              " or one of 'permanent', 'random', 'stable', 'preserve'.",
+                              " or one of 'permanent', 'random', 'stable', 'preserve', 'stable-ssid' (Wi-Fi only).",
                               scalar(node));
         }
     }

src/util-internal.h

@@ -136,7 +136,7 @@ gboolean
 _is_auth_key_management_psk(const NetplanAuthenticationSettings* auth);
 
 gboolean
-_is_macaddress_special_nm_option(const char* value);
+_is_macaddress_special_nm_option(const NetplanNetDefinition* netdef, const char* value);
 
 gboolean
 _is_macaddress_special_nd_option(const char* value);

src/util.c

@@ -1239,12 +1239,13 @@ _is_auth_key_management_psk(const NetplanAuthenticationSettings* auth)
 }
 
 gboolean
-_is_macaddress_special_nm_option(const char* value)
+_is_macaddress_special_nm_option(const NetplanNetDefinition* netdef, const char* value)
 {
     return (   !g_strcmp0(value, "preserve")
             || !g_strcmp0(value, "permanent")
             || !g_strcmp0(value, "random")
-            || !g_strcmp0(value, "stable"));
+            || !g_strcmp0(value, "stable")
+            || (!g_strcmp0(value, "stable-ssid") && netdef->type == NETPLAN_DEF_TYPE_WIFI));
 }
 
 gboolean

tests/cli/test_state.py

@@ -85,8 +85,8 @@ def test_query_nm(self, mock):
         mock.assert_called_with(['nmcli', '-t', '-f',
                                  'DEVICE,NAME,UUID,FILENAME,TYPE,AUTOCONNECT',
                                  'con', 'show'], text=True)
-        self.assertEqual(len(res), 1)
-        self.assertListEqual([itf.get('device') for itf in res], ['wlan0'])
+        self.assertEqual(len(res), 2)
+        self.assertListEqual([itf.get('device') for itf in res], ['wlan0', 'wlan1'])
 
     @patch('subprocess.check_output')
     def test_query_nm_fail(self, mock):
@@ -417,6 +417,61 @@ def test_json_nm_wlan0(self, networkctl_mock, nm_ssid_mock):
         self.assertEqual(len(json.get('dns_search')), 1)
         self.assertEqual(len(json.get('routes')), 6)
 
+    @patch('netplan_cli.cli.state.Interface.query_nm_ssid')
+    @patch('netplan_cli.cli.state.Interface.query_networkctl')
+    def test_json_nm_wlan0_2(self, networkctl_mock, nm_ssid_mock):
+        SSID = 'MYCON'
+        nm_ssid_mock.return_value = SSID
+        # networkctl mock output reduced to relevant lines
+        # Newer versions of systemd changed WiFi to Wi-Fi
+        # See systemd commit 8fff78a1dded105e1ee87bc66e29ef2fd61bf8c9
+        networkctl_mock.return_value = \
+            'Wi-Fi access point: {} (b4:fb:e4:75:c6:21)'.format(SSID)
+
+        data = next((itf for itf in yaml.safe_load(IPROUTE2) if itf['ifindex'] == 5), {})
+        nd = SystemConfigState.process_networkd(NETWORKD)
+        nm = SystemConfigState.process_nm(NMCLI)
+        dns = (DNS_ADDRESSES, DNS_SEARCH)
+        routes = (SystemConfigState.process_generic(ROUTE4), SystemConfigState.process_generic(ROUTE6))
+
+        itf = Interface(data, nd, nm, dns, routes)
+        _, json = itf.json()
+        self.assertEqual(json.get('ssid'), 'MYCON')
+
+    @patch('netplan_cli.cli.state.Interface.query_nm_ssid')
+    @patch('netplan_cli.cli.state.Interface.query_networkctl')
+    def test_json_nd_wlan1_non_ascii(self, networkctl_mock, nm_ssid_mock):
+        ND_SSID = '\\303\\241\\303\\251\\303\\255\\303\\263\\303\\272\\302\\242\\302\\242\\302\\242\\302\\243\\302\\243\\302\\243'
+        NM_SSID = 'áéíóú¢¢¢£££'
+        nm_ssid_mock.return_value = NM_SSID
+        # networkctl mock output reduced to relevant lines
+        # Newer versions of systemd changed WiFi to Wi-Fi
+        # See systemd commit 8fff78a1dded105e1ee87bc66e29ef2fd61bf8c9
+        networkctl_mock.return_value = \
+            'Wi-Fi access point: {} (b4:fb:e4:75:c6:21)'.format(ND_SSID)
+
+        data = {'ifname': 'wlan1', 'ifindex': 123}
+        nd = [{'Index': 123, 'Type': 'wlan', 'Name': 'wlan1', 'SetupState': 'managed',
+               'NetworkFile': '/run/systemd/network/10-netplan-wlan1.network'}]
+        nm = SystemConfigState.process_nm(NMCLI)
+
+        itf = Interface(data, nd, nm, (None, None), (None, None))
+        _, json = itf.json()
+        self.assertEqual(json.get('ssid'), NM_SSID)
+
+    @patch('netplan_cli.cli.state.Interface.query_nm_ssid')
+    def test_json_nm_wlan1_non_ascii(self, nm_ssid_mock):
+        NM_SSID = 'áéíóú¢¢¢£££'
+        nm_ssid_mock.return_value = NM_SSID
+
+        data = {'ifname': 'wlan1', 'ifindex': 123}
+        nd = [{'Index': 123, 'Type': 'wlan', 'Name': 'wlan1', 'SetupState': 'unmanaged'}]
+        nm = SystemConfigState.process_nm(NMCLI)
+
+        itf = Interface(data, nd, nm, (None, None), (None, None))
+        _, json = itf.json()
+        self.assertEqual(json.get('ssid'), NM_SSID)
+
     @patch('netplan_cli.cli.state.Interface.query_networkctl')
     def test_json_nd_enp0s31f6(self, networkctl_mock):
         # networkctl mock output reduced to relevant lines