chore: refactor "installing" resources to ~/.ptx

.devcontainer/postCreateCommand.sh

@@ -8,7 +8,7 @@ echo 'export PATH="/root/.local/bin:$PATH"' > ~/.bashrc
 sudo `which poetry` config virtualenvs.create false
 sudo `which poetry` install --with dev
 python scripts/fetch_core.py
-python scripts/zip_templates.py
+python scripts/bundle_resources.py
 playwright install-deps
 playwright install
 # Run mypy once so that it will install any needed type stubs. After this, the VSCode extension will run it automatically.

.github/workflows/tests.yml

@@ -20,7 +20,7 @@ jobs:
         run: |
           python -m pip install -e .
           python scripts/fetch_core.py
-          python scripts/zip_templates.py
+          python scripts/bundle_resources.py
       - name: Check formatting with black
         run: python -m black --check --diff $(git ls-files "*.py")
       - name: Check for lint
@@ -75,7 +75,7 @@ jobs:
         run: |
           python -m poetry install
           python -m poetry run python scripts/fetch_core.py
-          python -m poetry run python scripts/zip_templates.py
+          python -m poetry run python scripts/bundle_resources.py
 
       - name: Test with pytest
         run: |

.gitignore

@@ -138,11 +138,8 @@ cython_debug/
 
 #pretext-core
 pretext/core/pretext.py
-pretext/core/resources.zip
-#shipped templates
-pretext/templates/resources
-#old "static" stuff (deprecated
-pretext/static
+#zipped resources
+pretext/resources/*.zip
 
 #default new pretext project
 new-pretext-project

pretext/__init__.py

@@ -21,7 +21,7 @@
 
 VERSION = get_version("pretext", Path(__file__).parent.parent)
 
-CORE_COMMIT = "ed4e9d94a75b5b728aca2eb06dfde3f1595ab783"
+CORE_COMMIT = "ac1ca3ca67c9512059afd6fd37a714a7fc988a5d"
 
 
 def activate() -> None:

pretext/cli.py

@@ -25,7 +25,7 @@
 
 from . import (
     utils,
-    templates,
+    resources,
     core,
     constants,
     plastex,
@@ -248,7 +248,8 @@ def devscript(args: List[str]) -> None:
     """
     PY_CMD = sys.executable
     subprocess.run(
-        [PY_CMD, str(core.resources.path("pretext", "pretext"))] + list(args)
+        [PY_CMD, str(resources.resource_base_path() / "core" / "pretext" / "pretext")]
+        + list(args)
     )
 
 
@@ -284,47 +285,35 @@ def new(template: str, directory: Path, url_template: str) -> None:
     """
     directory_fullpath = Path(directory).resolve()
     if utils.project_path(directory_fullpath) is not None:
-        log.warning(
+        log.error(
             f"A project already exists in `{utils.project_path(directory_fullpath)}`."
         )
-        log.warning("No new project will be generated.")
+        log.error("No new project will be generated.")
         return
-    log.info(
-        f"Generating new PreTeXt project in `{directory_fullpath}` using `{template}` template."
-    )
+    log.info(f"Generating new PreTeXt project in `{directory_fullpath}`")
+    directory_fullpath.mkdir(exist_ok=True)
     if url_template is not None:
+        log.info(f"Using template at `{url_template}`")
+        # get project and extract to directory
         r = requests.get(url_template)
         archive = zipfile.ZipFile(io.BytesIO(r.content))
+        with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname:
+            archive.extractall(tmpdirname)
+            content_path = [Path(tmpdirname) / i for i in os.listdir(tmpdirname)][0]
+            shutil.copytree(content_path, directory_fullpath, dirs_exist_ok=True)
     else:
-        with templates.resource_path(f"{template}.zip") as template_path:
-            archive = zipfile.ZipFile(template_path)
-    # find (first) project.ptx to use as root of template
-    filenames = [Path(filepath).name for filepath in archive.namelist()]
-    project_ptx_index = filenames.index("project.ptx")
-    project_ptx_path = Path(archive.namelist()[project_ptx_index])
-    project_dir_path = project_ptx_path.parent
-    with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname:
-        temp_path = Path(tmpdirname) / "new-project"
-        temp_path.mkdir()
-        for filepath in [
-            filepath
-            for filepath in archive.namelist()
-            if project_dir_path in Path(filepath).parents
-        ]:
-            archive.extract(filepath, path=temp_path)
-        tmpsubdirname = temp_path / project_dir_path
-        shutil.copytree(tmpsubdirname, directory_fullpath, dirs_exist_ok=True)
-    # generate remaining boilerplate like requirements.txt
-    project = Project.parse(directory_fullpath)
-    project.generate_boilerplate(update_requirements=True)
-    if len(project.targets) == 0:
-        log.warning("The generated project has no targets!")
-    else:
-        target = project.targets[0]
-        log.info(f"Success! Open `{target.source_abspath()}` to edit your document")
-    log.info(
-        f"Then try to `pretext build` and `pretext view` from within `{directory_fullpath}`."
-    )
+        log.info(f"Using `{template}` template.")
+        # copy project from installed resources
+        with resources.resource_base_path() / "templates" / f"{template}" as template_path:
+            shutil.copytree(template_path, directory_fullpath, dirs_exist_ok=True)
+        # generate missing boilerplate
+        with utils.working_directory(directory_fullpath):
+            project_path = utils.project_path()
+            if project_path is None:
+                project = Project()
+            else:
+                project = Project.parse(project_path)
+            project.generate_boilerplate(update_requirements=True)
 
 
 # pretext init

pretext/core/__init__.py

@@ -7,10 +7,10 @@
         "Run `scripts/fetch_core.py` to grab a copy of pretex core.\n"
         "The original error message is: " + e.msg
     )
-from . import resources
+from .. import resources
 from .. import CORE_COMMIT, VERSION
 
-set_ptx_path(resources.path())
+set_ptx_path(resources.resource_base_path() / "core")
 
 
 def cli_build_message() -> str:

pretext/project/__init__.py

@@ -24,7 +24,7 @@
 from .. import codechat
 from .. import utils
 from .. import types as pt  # PreTeXt types
-from .. import templates
+from ..resources import resource_base_path
 from .. import VERSION
 
 
@@ -286,7 +286,7 @@ def post_validate(self) -> None:
             if not self.publication_abspath().exists():
                 # ... then use the CLI's built-in template file.
                 # TODO: this is wrong, since the returned path is only valid inside the context manager. Instead, need to enter the context here, then exit it when this class is deleted (also problematic).
-                with templates.resource_path("publication.ptx") as self.publication:
+                with resource_base_path() / "templates" / "publication.ptx" as self.publication:
                     pass
         # Otherwise, verify that the provided publication file exists. TODO: It is silly to check that all publication files exist.  We warn when they don't.  If the target we are calling has a non-existent publication file, then that error will be caught anyway.
         else:
@@ -1455,7 +1455,7 @@ def generate_boilerplate(
                         f"Your existing {resource} file has been backed up at {backup_resource_path}."
                     )
             if resource != "requirements.txt":
-                with templates.resource_path(resource) as resource_path:
+                with resource_base_path() / "templates" / resource as resource_path:
                     if (
                         not project_resource_path.exists()
                         or resource_path.read_text()

pretext/resources/__init__.py

@@ -0,0 +1,43 @@
+import importlib.resources
+import logging
+from pathlib import Path
+import shutil
+import zipfile
+
+from .. import VERSION, CORE_COMMIT
+
+log = logging.getLogger("ptxlogger")
+
+_RESOURCE_BASE_PATH = Path.home() / ".ptx" / VERSION
+
+
+def install(reinstall: bool = False) -> None:
+    if _RESOURCE_BASE_PATH.exists():
+        if reinstall:
+            log.info(f"Deleting existing resources at {_RESOURCE_BASE_PATH}")
+            shutil.rmtree(_RESOURCE_BASE_PATH)
+        else:
+            log.warning(f"Resources are already installed at {_RESOURCE_BASE_PATH}")
+            return
+    _RESOURCE_BASE_PATH.mkdir(parents=True)
+
+    log.info("Installing core resources")
+    with importlib.resources.path("pretext.resources", "core.zip") as static_zip:
+        with zipfile.ZipFile(static_zip, "r") as zip:
+            zip.extractall(path=_RESOURCE_BASE_PATH)
+        (_RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename(
+            _RESOURCE_BASE_PATH / "core"
+        )
+
+    log.info("Installing templates")
+    (_RESOURCE_BASE_PATH / "templates").mkdir()
+    with importlib.resources.path("pretext.resources", "templates.zip") as static_zip:
+        with zipfile.ZipFile(static_zip, "r") as zip:
+            zip.extractall(path=_RESOURCE_BASE_PATH / "templates")
+
+
+def resource_base_path() -> Path:
+    if not _RESOURCE_BASE_PATH.exists():
+        log.info(f"Installing resources to {_RESOURCE_BASE_PATH}")
+        install()
+    return _RESOURCE_BASE_PATH

pretext/utils.py

@@ -19,7 +19,7 @@
 from typing import Any, cast, List, Optional
 
 
-from . import core, templates, constants
+from . import core, constants, resources
 
 # Get access to logger
 log = logging.getLogger("ptxlogger")
@@ -94,7 +94,7 @@ def project_xml(dirpath: t.Optional[Path] = None) -> _ElementTree:
         dirpath = Path()  # current directory
     pp = project_path(dirpath)
     if pp is None:
-        with templates.resource_path("project.ptx") as project_manifest:
+        with resources.resource_base_path() / "templates" / "project.ptx" as project_manifest:
             return ET.parse(project_manifest)
     else:
         project_manifest = pp / "project.ptx"
@@ -173,7 +173,7 @@ def xml_syntax_is_valid(xmlfile: Path, root_tag: str = "pretext") -> bool:
 
 def xml_source_validates_against_schema(xmlfile: Path) -> bool:
     # get path to RelaxNG schema file:
-    schemarngfile = core.resources.path("schema", "pretext.rng")
+    schemarngfile = resources.resource_base_path() / "core" / "schema" / "pretext.rng"
 
     # Open schemafile for validation:
     relaxng = ET.RelaxNG(file=schemarngfile)
@@ -295,7 +295,9 @@ def copy_custom_xsl(xsl_path: Path, output_dir: Path) -> None:
     log.debug(f"Copying all files in {xsl_dir} to {output_dir}")
     shutil.copytree(xsl_dir, output_dir, dirs_exist_ok=True)
     log.debug(f"Copying core XSL to {output_dir}/core")
-    shutil.copytree(core.resources.path("xsl"), output_dir / "core")
+    shutil.copytree(
+        resources.resource_base_path() / "core" / "xsl", output_dir / "core"
+    )
 
 
 def check_executable(exec_name: str) -> Optional[str]:
@@ -439,7 +441,9 @@ def show_target_hints(
 
 
 def npm_install() -> None:
-    with working_directory(core.resources.path("script", "mjsre")):
+    with working_directory(
+        resources.resource_base_path() / "core" / "script" / "mjsre"
+    ):
         log.info("Attempting to install/update required node packages.")
         try:
             subprocess.run("npm install", shell=True)

scripts/build_package.py

@@ -1,6 +1,6 @@
 import subprocess
 import fetch_core
-import zip_templates
+import bundle_resources
 
 
 def main() -> None:
@@ -10,7 +10,7 @@ def main() -> None:
 
     # ensure up-to-date "static" resources
     fetch_core.main()
-    zip_templates.main()
+    bundle_resources.main()
 
     # Build package
     subprocess.run(["poetry", "build"], shell=True)

scripts/bundle_resources.py

@@ -0,0 +1,16 @@
+import shutil
+from pathlib import Path
+
+
+def main() -> None:
+    shutil.make_archive(
+        str(Path("pretext") / "resources" / "templates"), "zip", Path("templates")
+    )
+    print("Templates successfully zipped.")
+    # TODO: incorporate in pelican branch
+    # shutil.make_archive(str(Path("pretext") / "resources" / "pelican"), 'zip', Path("pelican"))
+    # print("Pelican resources successfully zipped.")
+
+
+if __name__ == "__main__":
+    main()