From f502d26242201ab2a513ca9895b954294d51be0d Mon Sep 17 00:00:00 2001 From: flora-five <72858916+flora-five@users.noreply.github.com> Date: Thu, 18 Dec 2025 13:42:43 +0000 Subject: [PATCH 01/10] Change order of notes in README to have the patterns examples follow the definition --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3d045ce0..9348d472 100644 --- a/README.md +++ b/README.md @@ -437,9 +437,9 @@ source_path = [ *Few notes:* - If you specify a source path as a string that references a folder and the runtime begins with `python` or `nodejs`, the build process will automatically build python and nodejs dependencies if `requirements.txt` or `package.json` file will be found in the source folder. If you want to customize this behavior, please use the object notation as explained below. +- If you use the `commands` option and chain multiple commands, only the exit code of last command will be checked for success. If you prefer to fail fast, start the commands with the bash option `set -e` or powershell option `$ErrorActionPreference="Stop"` - All arguments except `path` are optional. - `patterns` - List of Python regex filenames should satisfy. Default value is "include everything" which is equal to `patterns = [".*"]`. This can also be specified as multiline heredoc string (no comments allowed). Some examples of valid patterns: -- If you use the `commands` option and chain multiple commands, only the exit code of last command will be checked for success. If you prefer to fail fast, start the commands with the bash option `set -e` or powershell option `$ErrorActionPreference="Stop"` ```txt !.*/.*\.txt # Filter all txt files recursively From fe0592fd643552881ac37395901388c33f682ccc Mon Sep 17 00:00:00 2001 From: flora-five <72858916+flora-five@users.noreply.github.com> Date: Thu, 18 Dec 2025 13:43:58 +0000 Subject: [PATCH 02/10] Change the paths in the README examples to match the types of applications --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9348d472..84121425 100644 --- a/README.md +++ b/README.md @@ -414,7 +414,7 @@ source_path = [ npm_tmp_dir = "/tmp/dir/location" prefix_in_zip = "foo/bar1", }, { - path = "src/python-app3", + path = "src/nodejs-app2", commands = [ "npm install", ":zip" @@ -424,7 +424,7 @@ source_path = [ "node_modules/.+", # Include all node_modules ], }, { - path = "src/python-app3", + path = "src/go-app1", commands = ["go build"], patterns = < Date: Thu, 18 Dec 2025 13:45:04 +0000 Subject: [PATCH 03/10] Move the node-app fixture into the examples folder, where it is expected to be --- {tests => examples}/fixtures/node-app/index.js | 0 {tests => examples}/fixtures/node-app/package.json | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {tests => examples}/fixtures/node-app/index.js (100%) rename {tests => examples}/fixtures/node-app/package.json (100%) diff --git a/tests/fixtures/node-app/index.js b/examples/fixtures/node-app/index.js similarity index 100% rename from tests/fixtures/node-app/index.js rename to examples/fixtures/node-app/index.js diff --git a/tests/fixtures/node-app/package.json b/examples/fixtures/node-app/package.json similarity index 100% rename from tests/fixtures/node-app/package.json rename to examples/fixtures/node-app/package.json From b94d49fda734bb20ca0d4f61cd09b4e7e2257de0 Mon Sep 17 00:00:00 2001 From: flora-five <72858916+flora-five@users.noreply.github.com> Date: Thu, 18 Dec 2025 13:45:49 +0000 Subject: [PATCH 04/10] Avoid double encoding of source_path strings --- package.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.tf b/package.tf index 99078600..9e303ba6 100644 --- a/package.tf +++ b/package.tf @@ -28,7 +28,7 @@ data "external" "archive_prepare" { artifacts_dir = var.artifacts_dir runtime = var.runtime - source_path = jsonencode(var.source_path) + source_path = try(tostring(var.source_path), jsonencode(var.source_path)) hash_extra = var.hash_extra hash_extra_paths = jsonencode( [ From ebf03c41d88dc7a2bb1fe0576c9b43b0428eda05 Mon Sep 17 00:00:00 2001 From: flora-five <72858916+flora-five@users.noreply.github.com> Date: Thu, 18 Dec 2025 13:46:12 +0000 Subject: [PATCH 05/10] Normalize other paths used as input to the plan Previously, the following paths were normalized: the working directory for commands and the path elements when source_path is a list. With this change, the following paths are normalized too: plain paths given as source_path strings, path values of npm_requirements and pip_requirements. --- package.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package.py b/package.py index 3261a282..93b3635d 100644 --- a/package.py +++ b/package.py @@ -767,7 +767,6 @@ def commands_step(path, commands): commands = map(str.strip, commands.splitlines()) if path: - path = os.path.normpath(path) step("set:workdir", path) batch = [] @@ -805,7 +804,7 @@ def commands_step(path, commands): for claim in claims: if isinstance(claim, str): - path = claim + path = os.path.normpath(claim) if not os.path.exists(path): abort( 'Could not locate source_path "{path}". Paths are relative to directory where `terraform plan` is being run ("{pwd}")'.format( @@ -823,6 +822,8 @@ def commands_step(path, commands): elif isinstance(claim, dict): path = claim.get("path") + if path: + path = os.path.normpath(path) patterns = claim.get("patterns") commands = claim.get("commands") if patterns: @@ -849,7 +850,7 @@ def commands_step(path, commands): ) else: pip_requirements_step( - pip_requirements, + os.path.normpath(pip_requirements), prefix, required=True, tmp_dir=claim.get("pip_tmp_dir"), @@ -875,13 +876,12 @@ def commands_step(path, commands): ) else: npm_requirements_step( - npm_requirements, + os.path.normpath(npm_requirements), prefix, required=True, tmp_dir=claim.get("npm_tmp_dir"), ) if path: - path = os.path.normpath(path) step("zip", path, prefix) if patterns: # Take patterns into account when computing hash From aa193eb9527bb5f586b4040714931142d2a359dc Mon Sep 17 00:00:00 2001 From: flora-five <72858916+flora-five@users.noreply.github.com> Date: Thu, 18 Dec 2025 17:28:25 +0000 Subject: [PATCH 06/10] Include again into the hash the content of package.py, ignoring its path See PR #66 TODO: remove the code and logic for hash_extra_paths? --- package.py | 5 ++++- package.tf | 19 +++++++------------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/package.py b/package.py index 93b3635d..fc72b6e2 100644 --- a/package.py +++ b/package.py @@ -1691,7 +1691,7 @@ def prepare_command(args): if log.isEnabledFor(DEBUG3): log.debug("QUERY: %s", json.dumps(query_data, indent=2)) else: - log_excludes = ("source_path", "hash_extra_paths", "paths") + log_excludes = ("source_path", "hash_extra_paths", "hash_internal", "paths") qd = {k: v for k, v in query_data.items() if k not in log_excludes} log.debug("QUERY (excerpt): %s", json.dumps(qd, indent=2)) @@ -1704,6 +1704,7 @@ def prepare_command(args): hash_extra_paths = query.hash_extra_paths source_path = query.source_path hash_extra = query.hash_extra + hash_internal = query.hash_internal recreate_missing_package = yesno_bool( args.recreate_missing_package if args.recreate_missing_package is not None @@ -1723,6 +1724,8 @@ def prepare_command(args): content_hash = bpm.hash(hash_extra_paths) content_hash.update(json.dumps(build_plan, sort_keys=True).encode()) content_hash.update(runtime.encode()) + for c in hash_internal: + content_hash.update(c.encode()) content_hash.update(hash_extra.encode()) content_hash = content_hash.hexdigest() diff --git a/package.tf b/package.tf index 9e303ba6..cc6e39c4 100644 --- a/package.tf +++ b/package.tf @@ -26,18 +26,13 @@ data "external" "archive_prepare" { docker_entrypoint = var.docker_entrypoint }) : null - artifacts_dir = var.artifacts_dir - runtime = var.runtime - source_path = try(tostring(var.source_path), jsonencode(var.source_path)) - hash_extra = var.hash_extra - hash_extra_paths = jsonencode( - [ - # Temporary fix when building from multiple locations - # We should take into account content of package.py when counting hash - # Related issue: https://github.com/terraform-aws-modules/terraform-aws-lambda/issues/63 - # "${path.module}/package.py" - ] - ) + artifacts_dir = var.artifacts_dir + runtime = var.runtime + source_path = try(tostring(var.source_path), jsonencode(var.source_path)) + hash_extra = var.hash_extra + hash_extra_paths = jsonencode([]) + # Include into the hash the module sources that affect the packaging. + hash_internal = jsonencode([filesha256("${path.module}/package.py")]) recreate_missing_package = var.recreate_missing_package quiet = var.quiet_archive_local_exec From 3e5a6d5fa5fe3c4ec71d53fa9aa138afe23e142d Mon Sep 17 00:00:00 2001 From: flora-five <72858916+flora-five@users.noreply.github.com> Date: Thu, 18 Dec 2025 18:53:58 +0000 Subject: [PATCH 07/10] Filter the directories/files hashed when using commands with patterns --- package.py | 93 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 33 deletions(-) diff --git a/package.py b/package.py index fc72b6e2..a369dddb 100644 --- a/package.py +++ b/package.py @@ -243,32 +243,50 @@ def generate_content_hash(source_paths, hash_func=hashlib.sha256, log=None): if log: log = log.getChild("hash") + _log = log if log.isEnabledFor(DEBUG3) else None hash_obj = hash_func() - for source_path in source_paths: - if os.path.isdir(source_path): - source_dir = source_path - _log = log if log.isEnabledFor(DEBUG3) else None - for source_file in list_files(source_dir, log=_log): + for source_path, pf, prefix in source_paths: + if pf is not None: + for path_from_pattern in pf.filter(source_path, prefix): + if os.path.isdir(path_from_pattern): + # Hash only the path of the directory + source_dir = path_from_pattern + source_file = None + else: + source_dir = os.path.dirname(path_from_pattern) + source_file = os.path.relpath(path_from_pattern, source_dir) update_hash(hash_obj, source_dir, source_file) if log: - log.debug(os.path.join(source_dir, source_file)) + log.debug(path_from_pattern) else: - source_dir = os.path.dirname(source_path) - source_file = os.path.relpath(source_path, source_dir) - update_hash(hash_obj, source_dir, source_file) - if log: - log.debug(source_path) + if os.path.isdir(source_path): + source_dir = source_path + for source_file in list_files(source_dir, log=_log): + update_hash(hash_obj, source_dir, source_file) + if log: + log.debug(os.path.join(source_dir, source_file)) + else: + source_dir = os.path.dirname(source_path) + source_file = os.path.relpath(source_path, source_dir) + update_hash(hash_obj, source_dir, source_file) + if log: + log.debug(source_path) return hash_obj -def update_hash(hash_obj, file_root, file_path): +def update_hash(hash_obj, file_root, file_path=None): """ - Update a hashlib object with the relative path and contents of a file. + Update a hashlib object with the relative path and, if the given + file_path is not None, its content. """ + if file_path is None: + hash_obj.update(file_root.encode()) + return + relative_path = os.path.join(file_root, file_path) hash_obj.update(relative_path.encode()) @@ -562,7 +580,6 @@ class ZipContentFilter: def __init__(self, args): self._args = args self._rules = None - self._excludes = set() self._log = logging.getLogger("zip") def compile(self, patterns): @@ -668,7 +685,7 @@ def hash(self, extra_paths): if not self._source_paths: raise ValueError("BuildPlanManager.plan() should be called first") - content_hash_paths = self._source_paths + extra_paths + content_hash_paths = self._source_paths + [(p, None, None) for p in extra_paths] # Generate a hash based on file names and content. Also use the # runtime value, build command, and content of the build paths @@ -677,7 +694,7 @@ def hash(self, extra_paths): content_hash = generate_content_hash(content_hash_paths, log=self._log) return content_hash - def plan(self, source_path, query): + def plan(self, source_path, query, log=None): claims = source_path if not isinstance(source_path, list): claims = [source_path] @@ -686,11 +703,14 @@ def plan(self, source_path, query): build_plan = [] build_step = [] + if log: + log = log.getChild("plan") + def step(*x): build_step.append(x) - def hash(path): - source_paths.append(path) + def hash(path, patterns=None, prefix=None): + source_paths.append((path, patterns, prefix)) def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): command = runtime @@ -759,7 +779,7 @@ def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None): step("npm", runtime, requirements, prefix, tmp_dir) hash(requirements) - def commands_step(path, commands): + def commands_step(path, commands, patterns): if not commands: return @@ -773,8 +793,6 @@ def commands_step(path, commands): for c in commands: if isinstance(c, str): if c.startswith(":zip"): - if path: - hash(path) if batch: step("sh", "\n".join(batch)) batch.clear() @@ -785,12 +803,18 @@ def commands_step(path, commands): prefix = prefix.strip() _path = os.path.normpath(_path) step("zip:embedded", _path, prefix) + if path: + hash(path, patterns, prefix) elif n == 2: _, _path = c _path = os.path.normpath(_path) step("zip:embedded", _path) + if path: + hash(path, patterns=patterns) elif n == 1: step("zip:embedded") + if path: + hash(path, patterns=patterns) else: raise ValueError( ":zip invalid call signature, use: " @@ -829,7 +853,7 @@ def commands_step(path, commands): if patterns: step("set:filter", patterns_list(self._args, patterns)) if commands: - commands_step(path, commands) + commands_step(path, commands, patterns) else: prefix = claim.get("prefix_in_zip") pip_requirements = claim.get("pip_requirements") @@ -883,15 +907,7 @@ def commands_step(path, commands): ) if path: step("zip", path, prefix) - if patterns: - # Take patterns into account when computing hash - pf = ZipContentFilter(args=self._args) - pf.compile(patterns) - - for path_from_pattern in pf.filter(path, prefix): - hash(path_from_pattern) - else: - hash(path) + hash(path, patterns, prefix) else: raise ValueError("Unsupported source_path item: {}".format(claim)) @@ -899,7 +915,18 @@ def commands_step(path, commands): build_plan.append(build_step) build_step = [] - self._source_paths = source_paths + if log.isEnabledFor(DEBUG3): + log.debug("source_paths: %s", json.dumps(source_paths, indent=2)) + + for p, patterns, prefix in source_paths: + if self._source_paths is None: + self._source_paths = [] + pf = None + if patterns is not None: + pf = ZipContentFilter(args=self._args) + pf.compile(patterns) + self._source_paths.append((p, pf, prefix)) + return build_plan def execute(self, build_plan, zip_stream, query): @@ -1713,7 +1740,7 @@ def prepare_command(args): docker = query.docker bpm = BuildPlanManager(args, log=log) - build_plan = bpm.plan(source_path, query) + build_plan = bpm.plan(source_path, query, log) if log.isEnabledFor(DEBUG2): log.debug("BUILD_PLAN: %s", json.dumps(build_plan, indent=2)) From dcc7f7fd84a217e9f30e6fb686c42beaafbdbb85 Mon Sep 17 00:00:00 2001 From: flora-five <72858916+flora-five@users.noreply.github.com> Date: Thu, 18 Dec 2025 17:33:56 +0000 Subject: [PATCH 08/10] Update to the latest version of pre-commit-terraform --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 991a8bbf..c462b7f9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.103.0 + rev: v1.104.0 hooks: - id: terraform_fmt - id: terraform_wrapper_module_for_each From 97db8e3db9eba579e5af6720c8de8e68207ce084 Mon Sep 17 00:00:00 2001 From: flora-five <72858916+flora-five@users.noreply.github.com> Date: Thu, 18 Dec 2025 19:26:02 +0000 Subject: [PATCH 09/10] Update path to app source --- examples/simple-cicd/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple-cicd/main.tf b/examples/simple-cicd/main.tf index deefc9aa..93746f23 100644 --- a/examples/simple-cicd/main.tf +++ b/examples/simple-cicd/main.tf @@ -19,7 +19,7 @@ module "lambda_function" { runtime = "python3.12" source_path = [ - "${path.module}/src/python-app1", + "${path.module}/../fixtures/python-app1", ] trigger_on_package_timestamp = false } From 784f889950f997c4fa53501fa4f8b2d6f62e3fd5 Mon Sep 17 00:00:00 2001 From: flora-five <72858916+flora-five@users.noreply.github.com> Date: Thu, 18 Dec 2025 20:39:02 +0000 Subject: [PATCH 10/10] Update to version 6.x of terraform-aws-vpc to fix warnings about deprecated attributes with version 6.x of AWS provider --- examples/event-source-mapping/README.md | 2 +- examples/event-source-mapping/main.tf | 2 +- examples/with-efs/README.md | 2 +- examples/with-efs/main.tf | 2 +- examples/with-vpc-s3-endpoint/README.md | 4 ++-- examples/with-vpc-s3-endpoint/main.tf | 4 ++-- examples/with-vpc/README.md | 2 +- examples/with-vpc/main.tf | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/event-source-mapping/README.md b/examples/event-source-mapping/README.md index 49d4fa75..7b9d8e13 100644 --- a/examples/event-source-mapping/README.md +++ b/examples/event-source-mapping/README.md @@ -35,7 +35,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| | [lambda\_function](#module\_lambda\_function) | ../../ | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/event-source-mapping/main.tf b/examples/event-source-mapping/main.tf index f76d30c8..a55d1758 100644 --- a/examples/event-source-mapping/main.tf +++ b/examples/event-source-mapping/main.tf @@ -241,7 +241,7 @@ resource "aws_kinesis_stream" "this" { # Amazon MQ module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = random_pet.this.id cidr = local.vpc_cidr diff --git a/examples/with-efs/README.md b/examples/with-efs/README.md index ce9cc15e..408fd728 100644 --- a/examples/with-efs/README.md +++ b/examples/with-efs/README.md @@ -36,7 +36,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| | [lambda\_function\_with\_efs](#module\_lambda\_function\_with\_efs) | ../../ | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/with-efs/main.tf b/examples/with-efs/main.tf index 90a0abed..8912b4d3 100644 --- a/examples/with-efs/main.tf +++ b/examples/with-efs/main.tf @@ -44,7 +44,7 @@ module "lambda_function_with_efs" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = random_pet.this.id cidr = "10.10.0.0/16" diff --git a/examples/with-vpc-s3-endpoint/README.md b/examples/with-vpc-s3-endpoint/README.md index f84ba32c..36663ada 100644 --- a/examples/with-vpc-s3-endpoint/README.md +++ b/examples/with-vpc-s3-endpoint/README.md @@ -40,8 +40,8 @@ Note that this example may create resources which cost money. Run `terraform des | [lambda\_s3\_write](#module\_lambda\_s3\_write) | ../../ | n/a | | [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 5.0 | | [security\_group\_lambda](#module\_security\_group\_lambda) | terraform-aws-modules/security-group/aws | ~> 4.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | -| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | +| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 6.0 | ## Resources diff --git a/examples/with-vpc-s3-endpoint/main.tf b/examples/with-vpc-s3-endpoint/main.tf index 29de6eba..5b21dbc5 100644 --- a/examples/with-vpc-s3-endpoint/main.tf +++ b/examples/with-vpc-s3-endpoint/main.tf @@ -59,7 +59,7 @@ data "aws_ec2_managed_prefix_list" "this" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = random_pet.this.id cidr = "10.0.0.0/16" @@ -101,7 +101,7 @@ module "vpc" { module "vpc_endpoints" { source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" - version = "~> 5.0" + version = "~> 6.0" vpc_id = module.vpc.vpc_id diff --git a/examples/with-vpc/README.md b/examples/with-vpc/README.md index e1808811..efe9f3be 100644 --- a/examples/with-vpc/README.md +++ b/examples/with-vpc/README.md @@ -36,7 +36,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| | [lambda\_function\_in\_vpc](#module\_lambda\_function\_in\_vpc) | ../../ | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/with-vpc/main.tf b/examples/with-vpc/main.tf index d373d724..85817b52 100644 --- a/examples/with-vpc/main.tf +++ b/examples/with-vpc/main.tf @@ -30,7 +30,7 @@ module "lambda_function_in_vpc" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = random_pet.this.id cidr = "10.10.0.0/16"