reload only reloads changed files; see #106

Author: craigbarratt

custom_components/pyscript/__init__.py

@@ -150,12 +150,6 @@ async def reload_scripts_handler(call):
 
         global_ctx_only = call.data.get("global_ctx", None)
 
-        if global_ctx_only is not None and not GlobalContextMgr.get(global_ctx_only):
-            _LOGGER.error("pyscript.reload: no global context '%s' to reload", global_ctx_only)
-            return
-
-        await unload_scripts(global_ctx_only=global_ctx_only)
-
         await install_requirements(hass, config_entry, pyscript_folder)
         await load_scripts(hass, config_entry.data, global_ctx_only=global_ctx_only)
 
@@ -260,19 +254,47 @@ async def unload_scripts(global_ctx_only=None, unload_all=False):
         global_ctx.stop()
         ctx_delete[global_ctx_name] = global_ctx
     for global_ctx_name, global_ctx in ctx_delete.items():
-        await GlobalContextMgr.delete(global_ctx_name)
+        GlobalContextMgr.delete(global_ctx_name)
 
 
 @bind_hass
-async def load_scripts(hass, data, global_ctx_only=None):
+async def load_scripts(hass, config_data, global_ctx_only=None):
     """Load all python scripts in FOLDER."""
 
+    class SourceFile:
+        """Class for information about a source file."""
+
+        def __init__(
+            self,
+            global_ctx_name=None,
+            file_path=None,
+            rel_path=None,
+            rel_import_path=None,
+            fq_mod_name=None,
+            check_config=None,
+            app_config=None,
+            source=None,
+            mtime=None,
+            autoload=None,
+        ):
+            self.global_ctx_name = global_ctx_name
+            self.file_path = file_path
+            self.rel_path = rel_path
+            self.rel_import_path = rel_import_path
+            self.fq_mod_name = fq_mod_name
+            self.check_config = check_config
+            self.app_config = app_config
+            self.source = source
+            self.mtime = mtime
+            self.autoload = autoload
+            self.force = False
+
     pyscript_dir = hass.config.path(FOLDER)
 
-    def glob_files(load_paths, data):
-        source_files = []
-        apps_config = data.get("apps", None)
-        for path, match, check_config in load_paths:
+    def glob_read_files(load_paths, apps_config):
+        """Expand globs and read all the source files."""
+        ctx2source = {}
+        for path, match, check_config, autoload in load_paths:
             for this_path in sorted(glob.glob(os.path.join(pyscript_dir, path, match), recursive=True)):
                 rel_import_path = None
                 rel_path = this_path
@@ -281,12 +303,13 @@ def glob_files(load_paths, data):
                 if rel_path.startswith("/"):
                     rel_path = rel_path[1:]
                 if rel_path[0] == "#" or rel_path.find("/#") >= 0:
+                    # skip "commented" files and directories
                     continue
-                rel_path = rel_path[0:-3]
-                if rel_path.endswith("/__init__"):
-                    rel_path = rel_path[0 : -len("/__init__")]
-                    rel_import_path = rel_path
-                mod_name = rel_path.replace("/", ".")
+                mod_name = rel_path[0:-3]
+                if mod_name.endswith("/__init__"):
+                    # mod_name = mod_name[0 : -len("/__init__")]
+                    rel_import_path = mod_name
+                mod_name = mod_name.replace("/", ".")
                 if path == "":
                     global_ctx_name = f"file.{mod_name}"
                     fq_mod_name = mod_name
@@ -295,30 +318,199 @@ def glob_files(load_paths, data):
                     i = fq_mod_name.find(".")
                     if i >= 0:
                         fq_mod_name = fq_mod_name[i + 1 :]
+                app_config = None
+
+                if global_ctx_name in ctx2source:
+                    # the globs result in apps/APP/__init__.py matching twice, so skip the 2nd time
+                    continue
+
                 if check_config:
-                    if not isinstance(apps_config, dict) or fq_mod_name not in apps_config:
-                        _LOGGER.debug("load_scripts: skipping %s because config not present", this_path)
+                    app_name = fq_mod_name
+                    i = fq_mod_name.find(".")
+                    if i >= 0:
+                        app_name = app_name[0:i]
+                    if not isinstance(apps_config, dict) or app_name not in apps_config:
+                        _LOGGER.debug(
+                            "load_scripts: skipping %s (app_name=%s) because config not present",
+                            this_path,
+                            app_name,
+                        )
                         continue
-                source_files.append([global_ctx_name, this_path, rel_import_path, fq_mod_name])
+                    app_config = apps_config[app_name]
+
+                try:
+                    with open(this_path) as file_desc:
+                        source = file_desc.read()
+                    mtime = os.path.getmtime(this_path)
+                except Exception as exc:
+                    _LOGGER.error("load_scripts: skipping %s due to exception %s", this_path, exc)
+                    continue
 
-        return source_files
+                ctx2source[global_ctx_name] = SourceFile(
+                    global_ctx_name=global_ctx_name,
+                    file_path=this_path,
+                    rel_path=rel_path,
+                    rel_import_path=rel_import_path,
+                    fq_mod_name=fq_mod_name,
+                    check_config=check_config,
+                    app_config=app_config,
+                    source=source,
+                    mtime=mtime,
+                    autoload=autoload,
+                )
+
+        return ctx2source
 
     load_paths = [
-        ["apps", "*.py", True],
-        ["apps", "*/__init__.py", True],
-        ["", "*.py", False],
-        ["scripts", "**/*.py", False],
+        # directory, glob, check_config, autoload
+        ["", "*.py", False, True],
+        ["apps", "*.py", True, True],
+        ["apps", "*/__init__.py", True, True],
+        ["apps", "*/**/*.py", False, False],
+        ["modules", "*.py", False, False],
+        ["modules", "*/**/*.py", False, False],
+        ["scripts", "**/*.py", False, True],
     ]
 
-    source_files = await hass.async_add_executor_job(glob_files, load_paths, data)
-    for global_ctx_name, source_file, rel_import_path, fq_mod_name in source_files:
-        if global_ctx_only is not None:
-            if global_ctx_name != global_ctx_only and not global_ctx_name.startswith(global_ctx_only + "."):
-                continue
+    #
+    # get current global contexts
+    #
+    ctx_all = {}
+    for global_ctx_name, global_ctx in GlobalContextMgr.items():
+        idx = global_ctx_name.find(".")
+        if idx < 0 or global_ctx_name[0:idx] not in {"file", "apps", "modules", "scripts"}:
+            continue
+        ctx_all[global_ctx_name] = global_ctx
+
+    #
+    # get list and contents of all source files
+    #
+    apps_config = config_data.get("apps", None)
+    ctx2files = await hass.async_add_executor_job(glob_read_files, load_paths, apps_config)
+
+    #
+    # figure out what to reload based on global_ctx_only and what's changed
+    #
+    ctx_delete = set()
+    if global_ctx_only is not None and global_ctx_only != "*":
+        if global_ctx_only not in ctx_all and global_ctx_only not in ctx2files:
+            _LOGGER.error("pyscript.reload: no global context '%s' to reload", global_ctx_only)
+            return
+        if global_ctx_only not in ctx2files:
+            ctx_delete.add(global_ctx_only)
+        else:
+            ctx2files[global_ctx_only].force = True
+    elif global_ctx_only == "*":
+        ctx_delete = set(ctx_all.keys())
+        for _, src_info in ctx2files.items():
+            src_info.force = True
+    else:
+        # delete all global_ctxs that aren't present in current files
+        for global_ctx_name, global_ctx in ctx_all.items():
+            if global_ctx_name not in ctx2files:
+                ctx_delete.add(global_ctx_name)
+        # delete all global_ctxs that have changeed source or mtime
+        for global_ctx_name, src_info in ctx2files.items():
+            if global_ctx_name in ctx_all:
+                ctx = ctx_all[global_ctx_name]
+                if (
+                    src_info.source != ctx.get_source()
+                    or src_info.app_config != ctx.get_app_config()
+                    or src_info.mtime != ctx.get_mtime()
+                ):
+                    ctx_delete.add(global_ctx_name)
+                    src_info.force = True
+            else:
+                src_info.force = src_info.autoload
+
+    #
+    # force reload if any files uses a module that is bring reloaded by
+    # recursively following each import; first find which modules are
+    # being reloaded
+    #
+    will_reload = set()
+    for global_ctx_name, src_info in ctx2files.items():
+        if global_ctx_name.startswith("modules.") and (global_ctx_name in ctx_delete or src_info.force):
+            parts = global_ctx_name.split(".")
+            root = f"{parts[0]}.{parts[1]}"
+            will_reload.add(root)
+
+    if len(will_reload) > 0:
+
+        def import_recurse(ctx_name, visited, ctx2imports):
+            if ctx_name in visited or ctx_name in ctx2imports:
+                return ctx2imports.get(ctx_name, set())
+            visited.add(ctx_name)
+            ctx = GlobalContextMgr.get(ctx_name)
+            if not ctx:
+                return set()
+            ctx2imports[ctx_name] = set()
+            for imp_name in ctx.get_imports():
+                ctx2imports[ctx_name].add(imp_name)
+                ctx2imports[ctx_name].update(import_recurse(imp_name, visited, ctx2imports))
+            return ctx2imports[ctx_name]
+
+        ctx2imports = {}
+        for global_ctx_name, global_ctx in ctx_all.items():
+            if global_ctx_name not in ctx2imports:
+                visited = set()
+                import_recurse(global_ctx_name, visited, ctx2imports)
+            for mod_name in ctx2imports.get(global_ctx_name, set()):
+                parts = mod_name.split(".")
+                root = f"{parts[0]}.{parts[1]}"
+                if root in will_reload:
+                    ctx_delete.add(global_ctx_name)
+                    if global_ctx_name in ctx2files:
+                        ctx2files[global_ctx_name].force = True
+
+    #
+    # if any file in an app or module has changed, then reload just the top-level
+    # __init__.py or module/app .py file, and delete everything else
+    #
+    done = set()
+    for global_ctx_name, src_info in ctx2files.items():
+        if not src_info.force:
+            continue
+        if not global_ctx_name.startswith("apps.") and not global_ctx_name.startswith("modules."):
+            continue
+        parts = global_ctx_name.split(".")
+        root = f"{parts[0]}.{parts[1]}"
+        if root in done:
+            continue
+        pkg_path = f"{parts[0]}/{parts[1]}/__init__.py"
+        mod_path = f"{parts[0]}/{parts[1]}.py"
+        for ctx_name, this_src_info in ctx2files.items():
+            if ctx_name == root or ctx_name.startswith(f"{root}."):
+                if this_src_info.rel_path in {pkg_path, mod_path}:
+                    this_src_info.force = True
+                else:
+                    this_src_info.force = False
+                ctx_delete.add(ctx_name)
+        done.add(root)
+
+    #
+    # delete contexts that are no longer needed or will be reloaded
+    #
+    for global_ctx_name in ctx_delete:
+        if global_ctx_name in ctx_all:
+            global_ctx = ctx_all[global_ctx_name]
+            global_ctx.stop()
+            _LOGGER.debug("reload: deleting global_ctx=%s", global_ctx_name)
+            GlobalContextMgr.delete(global_ctx_name)
+
+    #
+    # now load the requested files, and files that depend on loaded files
+    #
+    for global_ctx_name, src_info in sorted(ctx2files.items()):
+        if not src_info.autoload or not src_info.force:
+            continue
         global_ctx = GlobalContext(
-            global_ctx_name,
-            global_sym_table={"__name__": fq_mod_name},
+            src_info.global_ctx_name,
+            global_sym_table={"__name__": src_info.fq_mod_name},
             manager=GlobalContextMgr,
-            rel_import_path=rel_import_path,
+            rel_import_path=src_info.rel_import_path,
+            app_config=src_info.app_config,
+            source=src_info.source,
+            mtime=src_info.mtime,
         )
-        await GlobalContextMgr.load_file(source_file, global_ctx)
+        await GlobalContextMgr.load_file(global_ctx, src_info.file_path, source=src_info.source, force=True)