diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 991a8bbf..c462b7f9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.103.0 + rev: v1.104.0 hooks: - id: terraform_fmt - id: terraform_wrapper_module_for_each diff --git a/README.md b/README.md index 3d045ce0..84121425 100644 --- a/README.md +++ b/README.md @@ -414,7 +414,7 @@ source_path = [ npm_tmp_dir = "/tmp/dir/location" prefix_in_zip = "foo/bar1", }, { - path = "src/python-app3", + path = "src/nodejs-app2", commands = [ "npm install", ":zip" @@ -424,7 +424,7 @@ source_path = [ "node_modules/.+", # Include all node_modules ], }, { - path = "src/python-app3", + path = "src/go-app1", commands = ["go build"], patterns = < [lambda\_function](#module\_lambda\_function) | ../../ | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/event-source-mapping/main.tf b/examples/event-source-mapping/main.tf index f76d30c8..a55d1758 100644 --- a/examples/event-source-mapping/main.tf +++ b/examples/event-source-mapping/main.tf @@ -241,7 +241,7 @@ resource "aws_kinesis_stream" "this" { # Amazon MQ module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = random_pet.this.id cidr = local.vpc_cidr diff --git a/tests/fixtures/node-app/index.js b/examples/fixtures/node-app/index.js similarity index 100% rename from tests/fixtures/node-app/index.js rename to examples/fixtures/node-app/index.js diff --git a/tests/fixtures/node-app/package.json b/examples/fixtures/node-app/package.json similarity index 100% rename from tests/fixtures/node-app/package.json rename to examples/fixtures/node-app/package.json diff --git a/examples/simple-cicd/main.tf b/examples/simple-cicd/main.tf index deefc9aa..93746f23 100644 --- a/examples/simple-cicd/main.tf +++ b/examples/simple-cicd/main.tf @@ -19,7 +19,7 @@ module "lambda_function" { runtime = "python3.12" source_path = [ - "${path.module}/src/python-app1", + "${path.module}/../fixtures/python-app1", ] trigger_on_package_timestamp = false } diff --git a/examples/with-efs/README.md b/examples/with-efs/README.md index ce9cc15e..408fd728 100644 --- a/examples/with-efs/README.md +++ b/examples/with-efs/README.md @@ -36,7 +36,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| | [lambda\_function\_with\_efs](#module\_lambda\_function\_with\_efs) | ../../ | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/with-efs/main.tf b/examples/with-efs/main.tf index 90a0abed..8912b4d3 100644 --- a/examples/with-efs/main.tf +++ b/examples/with-efs/main.tf @@ -44,7 +44,7 @@ module "lambda_function_with_efs" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = random_pet.this.id cidr = "10.10.0.0/16" diff --git a/examples/with-vpc-s3-endpoint/README.md b/examples/with-vpc-s3-endpoint/README.md index f84ba32c..36663ada 100644 --- a/examples/with-vpc-s3-endpoint/README.md +++ b/examples/with-vpc-s3-endpoint/README.md @@ -40,8 +40,8 @@ Note that this example may create resources which cost money. Run `terraform des | [lambda\_s3\_write](#module\_lambda\_s3\_write) | ../../ | n/a | | [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 5.0 | | [security\_group\_lambda](#module\_security\_group\_lambda) | terraform-aws-modules/security-group/aws | ~> 4.0 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | -| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | +| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 6.0 | ## Resources diff --git a/examples/with-vpc-s3-endpoint/main.tf b/examples/with-vpc-s3-endpoint/main.tf index 29de6eba..5b21dbc5 100644 --- a/examples/with-vpc-s3-endpoint/main.tf +++ b/examples/with-vpc-s3-endpoint/main.tf @@ -59,7 +59,7 @@ data "aws_ec2_managed_prefix_list" "this" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = random_pet.this.id cidr = "10.0.0.0/16" @@ -101,7 +101,7 @@ module "vpc" { module "vpc_endpoints" { source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" - version = "~> 5.0" + version = "~> 6.0" vpc_id = module.vpc.vpc_id diff --git a/examples/with-vpc/README.md b/examples/with-vpc/README.md index e1808811..efe9f3be 100644 --- a/examples/with-vpc/README.md +++ b/examples/with-vpc/README.md @@ -36,7 +36,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Source | Version | |------|--------|---------| | [lambda\_function\_in\_vpc](#module\_lambda\_function\_in\_vpc) | ../../ | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 6.0 | ## Resources diff --git a/examples/with-vpc/main.tf b/examples/with-vpc/main.tf index d373d724..85817b52 100644 --- a/examples/with-vpc/main.tf +++ b/examples/with-vpc/main.tf @@ -30,7 +30,7 @@ module "lambda_function_in_vpc" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" + version = "~> 6.0" name = random_pet.this.id cidr = "10.10.0.0/16" diff --git a/package.py b/package.py index 3261a282..a369dddb 100644 --- a/package.py +++ b/package.py @@ -243,32 +243,50 @@ def generate_content_hash(source_paths, hash_func=hashlib.sha256, log=None): if log: log = log.getChild("hash") + _log = log if log.isEnabledFor(DEBUG3) else None hash_obj = hash_func() - for source_path in source_paths: - if os.path.isdir(source_path): - source_dir = source_path - _log = log if log.isEnabledFor(DEBUG3) else None - for source_file in list_files(source_dir, log=_log): + for source_path, pf, prefix in source_paths: + if pf is not None: + for path_from_pattern in pf.filter(source_path, prefix): + if os.path.isdir(path_from_pattern): + # Hash only the path of the directory + source_dir = path_from_pattern + source_file = None + else: + source_dir = os.path.dirname(path_from_pattern) + source_file = os.path.relpath(path_from_pattern, source_dir) update_hash(hash_obj, source_dir, source_file) if log: - log.debug(os.path.join(source_dir, source_file)) + log.debug(path_from_pattern) else: - source_dir = os.path.dirname(source_path) - source_file = os.path.relpath(source_path, source_dir) - update_hash(hash_obj, source_dir, source_file) - if log: - log.debug(source_path) + if os.path.isdir(source_path): + source_dir = source_path + for source_file in list_files(source_dir, log=_log): + update_hash(hash_obj, source_dir, source_file) + if log: + log.debug(os.path.join(source_dir, source_file)) + else: + source_dir = os.path.dirname(source_path) + source_file = os.path.relpath(source_path, source_dir) + update_hash(hash_obj, source_dir, source_file) + if log: + log.debug(source_path) return hash_obj -def update_hash(hash_obj, file_root, file_path): +def update_hash(hash_obj, file_root, file_path=None): """ - Update a hashlib object with the relative path and contents of a file. + Update a hashlib object with the relative path and, if the given + file_path is not None, its content. """ + if file_path is None: + hash_obj.update(file_root.encode()) + return + relative_path = os.path.join(file_root, file_path) hash_obj.update(relative_path.encode()) @@ -562,7 +580,6 @@ class ZipContentFilter: def __init__(self, args): self._args = args self._rules = None - self._excludes = set() self._log = logging.getLogger("zip") def compile(self, patterns): @@ -668,7 +685,7 @@ def hash(self, extra_paths): if not self._source_paths: raise ValueError("BuildPlanManager.plan() should be called first") - content_hash_paths = self._source_paths + extra_paths + content_hash_paths = self._source_paths + [(p, None, None) for p in extra_paths] # Generate a hash based on file names and content. Also use the # runtime value, build command, and content of the build paths @@ -677,7 +694,7 @@ def hash(self, extra_paths): content_hash = generate_content_hash(content_hash_paths, log=self._log) return content_hash - def plan(self, source_path, query): + def plan(self, source_path, query, log=None): claims = source_path if not isinstance(source_path, list): claims = [source_path] @@ -686,11 +703,14 @@ def plan(self, source_path, query): build_plan = [] build_step = [] + if log: + log = log.getChild("plan") + def step(*x): build_step.append(x) - def hash(path): - source_paths.append(path) + def hash(path, patterns=None, prefix=None): + source_paths.append((path, patterns, prefix)) def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None): command = runtime @@ -759,7 +779,7 @@ def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None): step("npm", runtime, requirements, prefix, tmp_dir) hash(requirements) - def commands_step(path, commands): + def commands_step(path, commands, patterns): if not commands: return @@ -767,15 +787,12 @@ def commands_step(path, commands): commands = map(str.strip, commands.splitlines()) if path: - path = os.path.normpath(path) step("set:workdir", path) batch = [] for c in commands: if isinstance(c, str): if c.startswith(":zip"): - if path: - hash(path) if batch: step("sh", "\n".join(batch)) batch.clear() @@ -786,12 +803,18 @@ def commands_step(path, commands): prefix = prefix.strip() _path = os.path.normpath(_path) step("zip:embedded", _path, prefix) + if path: + hash(path, patterns, prefix) elif n == 2: _, _path = c _path = os.path.normpath(_path) step("zip:embedded", _path) + if path: + hash(path, patterns=patterns) elif n == 1: step("zip:embedded") + if path: + hash(path, patterns=patterns) else: raise ValueError( ":zip invalid call signature, use: " @@ -805,7 +828,7 @@ def commands_step(path, commands): for claim in claims: if isinstance(claim, str): - path = claim + path = os.path.normpath(claim) if not os.path.exists(path): abort( 'Could not locate source_path "{path}". Paths are relative to directory where `terraform plan` is being run ("{pwd}")'.format( @@ -823,12 +846,14 @@ def commands_step(path, commands): elif isinstance(claim, dict): path = claim.get("path") + if path: + path = os.path.normpath(path) patterns = claim.get("patterns") commands = claim.get("commands") if patterns: step("set:filter", patterns_list(self._args, patterns)) if commands: - commands_step(path, commands) + commands_step(path, commands, patterns) else: prefix = claim.get("prefix_in_zip") pip_requirements = claim.get("pip_requirements") @@ -849,7 +874,7 @@ def commands_step(path, commands): ) else: pip_requirements_step( - pip_requirements, + os.path.normpath(pip_requirements), prefix, required=True, tmp_dir=claim.get("pip_tmp_dir"), @@ -875,23 +900,14 @@ def commands_step(path, commands): ) else: npm_requirements_step( - npm_requirements, + os.path.normpath(npm_requirements), prefix, required=True, tmp_dir=claim.get("npm_tmp_dir"), ) if path: - path = os.path.normpath(path) step("zip", path, prefix) - if patterns: - # Take patterns into account when computing hash - pf = ZipContentFilter(args=self._args) - pf.compile(patterns) - - for path_from_pattern in pf.filter(path, prefix): - hash(path_from_pattern) - else: - hash(path) + hash(path, patterns, prefix) else: raise ValueError("Unsupported source_path item: {}".format(claim)) @@ -899,7 +915,18 @@ def commands_step(path, commands): build_plan.append(build_step) build_step = [] - self._source_paths = source_paths + if log.isEnabledFor(DEBUG3): + log.debug("source_paths: %s", json.dumps(source_paths, indent=2)) + + for p, patterns, prefix in source_paths: + if self._source_paths is None: + self._source_paths = [] + pf = None + if patterns is not None: + pf = ZipContentFilter(args=self._args) + pf.compile(patterns) + self._source_paths.append((p, pf, prefix)) + return build_plan def execute(self, build_plan, zip_stream, query): @@ -1691,7 +1718,7 @@ def prepare_command(args): if log.isEnabledFor(DEBUG3): log.debug("QUERY: %s", json.dumps(query_data, indent=2)) else: - log_excludes = ("source_path", "hash_extra_paths", "paths") + log_excludes = ("source_path", "hash_extra_paths", "hash_internal", "paths") qd = {k: v for k, v in query_data.items() if k not in log_excludes} log.debug("QUERY (excerpt): %s", json.dumps(qd, indent=2)) @@ -1704,6 +1731,7 @@ def prepare_command(args): hash_extra_paths = query.hash_extra_paths source_path = query.source_path hash_extra = query.hash_extra + hash_internal = query.hash_internal recreate_missing_package = yesno_bool( args.recreate_missing_package if args.recreate_missing_package is not None @@ -1712,7 +1740,7 @@ def prepare_command(args): docker = query.docker bpm = BuildPlanManager(args, log=log) - build_plan = bpm.plan(source_path, query) + build_plan = bpm.plan(source_path, query, log) if log.isEnabledFor(DEBUG2): log.debug("BUILD_PLAN: %s", json.dumps(build_plan, indent=2)) @@ -1723,6 +1751,8 @@ def prepare_command(args): content_hash = bpm.hash(hash_extra_paths) content_hash.update(json.dumps(build_plan, sort_keys=True).encode()) content_hash.update(runtime.encode()) + for c in hash_internal: + content_hash.update(c.encode()) content_hash.update(hash_extra.encode()) content_hash = content_hash.hexdigest() diff --git a/package.tf b/package.tf index 99078600..cc6e39c4 100644 --- a/package.tf +++ b/package.tf @@ -26,18 +26,13 @@ data "external" "archive_prepare" { docker_entrypoint = var.docker_entrypoint }) : null - artifacts_dir = var.artifacts_dir - runtime = var.runtime - source_path = jsonencode(var.source_path) - hash_extra = var.hash_extra - hash_extra_paths = jsonencode( - [ - # Temporary fix when building from multiple locations - # We should take into account content of package.py when counting hash - # Related issue: https://github.com/terraform-aws-modules/terraform-aws-lambda/issues/63 - # "${path.module}/package.py" - ] - ) + artifacts_dir = var.artifacts_dir + runtime = var.runtime + source_path = try(tostring(var.source_path), jsonencode(var.source_path)) + hash_extra = var.hash_extra + hash_extra_paths = jsonencode([]) + # Include into the hash the module sources that affect the packaging. + hash_internal = jsonencode([filesha256("${path.module}/package.py")]) recreate_missing_package = var.recreate_missing_package quiet = var.quiet_archive_local_exec