Merge branch 'master' into refactor2

Gemfile.lock

@@ -9,14 +9,14 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.0)
-    aws-partitions (1.335.0)
+    aws-partitions (1.338.0)
-    aws-sdk-apigateway (1.47.0)
+    aws-sdk-apigateway (1.48.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-apigatewayv2 (1.23.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-athena (1.29.0)
+    aws-sdk-athena (1.30.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-autoscaling (1.22.0)
@@ -25,136 +25,136 @@ GEM
     aws-sdk-budgets (1.32.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudformation (1.39.0)
+    aws-sdk-cloudformation (1.41.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudfront (1.32.0)
+    aws-sdk-cloudfront (1.33.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-cloudhsm (1.24.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudhsmv2 (1.25.0)
+    aws-sdk-cloudhsmv2 (1.26.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudtrail (1.25.0)
+    aws-sdk-cloudtrail (1.26.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudwatch (1.40.0)
+    aws-sdk-cloudwatch (1.41.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-cloudwatchlogs (1.33.0)
+    aws-sdk-cloudwatchlogs (1.34.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-codecommit (1.36.0)
+    aws-sdk-codecommit (1.37.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-codedeploy (1.33.0)
+    aws-sdk-codedeploy (1.34.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-codepipeline (1.33.0)
+    aws-sdk-codepipeline (1.34.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-configservice (1.47.0)
+    aws-sdk-configservice (1.48.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-core (3.102.1)
+    aws-sdk-core (3.103.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-costandusagereportservice (1.23.0)
+    aws-sdk-costandusagereportservice (1.24.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-dynamodb (1.50.0)
+    aws-sdk-dynamodb (1.51.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ec2 (1.171.0)
+    aws-sdk-ec2 (1.174.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ecr (1.32.0)
+    aws-sdk-ecr (1.34.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ecs (1.66.0)
+    aws-sdk-ecs (1.67.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-efs (1.31.0)
+    aws-sdk-efs (1.32.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-eks (1.39.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticache (1.39.0)
+    aws-sdk-elasticache (1.40.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticbeanstalk (1.33.0)
+    aws-sdk-elasticbeanstalk (1.34.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticloadbalancing (1.24.0)
+    aws-sdk-elasticloadbalancing (1.25.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticloadbalancingv2 (1.46.0)
+    aws-sdk-elasticloadbalancingv2 (1.47.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-elasticsearchservice (1.38.0)
+    aws-sdk-elasticsearchservice (1.39.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-firehose (1.30.0)
+    aws-sdk-firehose (1.31.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-iam (1.42.0)
+    aws-sdk-iam (1.43.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-kafka (1.23.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kinesis (1.25.0)
+    aws-sdk-kinesis (1.26.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.35.0)
+    aws-sdk-kms (1.36.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-lambda (1.45.0)
+    aws-sdk-lambda (1.46.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-organizations (1.17.0)
       aws-sdk-core (~> 3, >= 3.39.0)
       aws-sigv4 (~> 1.0)
-    aws-sdk-rds (1.89.0)
+    aws-sdk-rds (1.92.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-redshift (1.45.0)
+    aws-sdk-redshift (1.46.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53 (1.39.0)
+    aws-sdk-route53 (1.40.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53domains (1.24.0)
+    aws-sdk-route53domains (1.25.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-route53resolver (1.16.0)
+    aws-sdk-route53resolver (1.17.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.71.1)
+    aws-sdk-s3 (1.73.0)
       aws-sdk-core (~> 3, >= 3.102.1)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.1)
-    aws-sdk-securityhub (1.28.0)
+    aws-sdk-securityhub (1.29.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ses (1.32.0)
+    aws-sdk-ses (1.33.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sms (1.22.0)
+    aws-sdk-sms (1.23.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sns (1.26.0)
+    aws-sdk-sns (1.27.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.29.0)
+    aws-sdk-sqs (1.30.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-ssm (1.83.0)
+    aws-sdk-ssm (1.84.0)
       aws-sdk-core (~> 3, >= 3.99.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.1)
@@ -167,13 +167,13 @@ GEM
       ms_rest_azure (~> 0.12.0)
     azure_mgmt_security (0.18.2)
       ms_rest_azure (~> 0.12.0)
-    azure_mgmt_storage (0.21.1)
+    azure_mgmt_storage (0.21.2)
       ms_rest_azure (~> 0.12.0)
     bcrypt_pbkdf (1.0.1)
     builder (3.2.4)
-    chef-config (16.2.50)
+    chef-config (16.2.73)
       addressable
-      chef-utils (= 16.2.50)
+      chef-utils (= 16.2.73)
       fuzzyurl
       mixlib-config (>= 2.2.12, < 4.0)
       mixlib-shellout (>= 2.0, < 4.0)
@@ -182,12 +182,12 @@ GEM
       chef-config
       concurrent-ruby (~> 1.0)
       ffi-yajl (~> 2.2)
-    chef-utils (16.2.50)
+    chef-utils (16.2.73)
     coderay (1.1.3)
     concurrent-ruby (1.1.6)
-    declarative (0.0.10)
+    declarative (0.0.20)
     declarative-option (0.1.0)
-    diff-lcs (1.4.2)
+    diff-lcs (1.4.4)
     docker-api (1.34.2)
       excon (>= 0.47.0)
       multi_json
@@ -238,14 +238,14 @@ GEM
     i18n (1.8.3)
       concurrent-ruby (~> 1.0)
     inifile (3.0.0)
-    inspec (4.20.10)
+    inspec (4.21.3)
       faraday_middleware (~> 0.12.2)
-      inspec-core (= 4.20.10)
+      inspec-core (= 4.21.3)
       train (~> 3.0)
       train-aws (~> 0.1)
       train-habitat (~> 0.1)
       train-winrm (~> 0.2)
-    inspec-core (4.20.10)
+    inspec-core (4.21.3)
       addressable (~> 2.4)
       chef-telemetry (~> 1.0)
       faraday (>= 0.9.0)
@@ -271,7 +271,7 @@ GEM
       tty-prompt (~> 0.17)
       tty-table (~> 0.10)
     jmespath (1.4.0)
-    json (2.3.0)
+    json (2.3.1)
     json_schemer (0.2.11)
       ecma-re-validator (~> 0.2)
       hana (~> 1.3)
@@ -294,9 +294,9 @@ GEM
       tty-box (~> 0.3)
       tty-prompt (~> 0.18)
     little-plugger (1.1.4)
-    logging (2.2.2)
+    logging (2.3.0)
       little-plugger (~> 1.1)
-      multi_json (~> 1.10)
+      multi_json (~> 1.14)
     memoist (0.16.2)
     method_source (1.0.0)
     mini_mime (1.0.2)
@@ -396,7 +396,7 @@ GEM
     tins (1.25.0)
       sync
     tomlrb (1.2.9)
-    train (3.3.4)
+    train (3.3.6)
       activesupport (>= 5.2.4.3, < 6.0.0)
       azure_graph_rbac (~> 0.16)
       azure_mgmt_key_vault (~> 0.17)
@@ -407,7 +407,7 @@ GEM
       google-api-client (>= 0.23.9, < 0.35.0)
       googleauth (>= 0.6.6, < 0.11.0)
       inifile (~> 3.0)
-      train-core (= 3.3.4)
+      train-core (= 3.3.6)
       train-winrm (~> 0.2)
     train-aws (0.1.17)
       aws-sdk-apigateway (~> 1.0)
@@ -458,7 +458,7 @@ GEM
       aws-sdk-sns (~> 1.9)
       aws-sdk-sqs (~> 1.10)
       aws-sdk-ssm (~> 1.0)
-    train-core (3.3.4)
+    train-core (3.3.6)
       addressable (~> 2.5)
       ffi (!= 1.13.0)
       json (>= 1.8, < 3.0)

prometheus/libtofs.jinja

@@ -1,40 +1,37 @@
-{%- macro files_switch(source_files,
+{%- macro files_switch(
+      source_files,
       lookup=None,
-                       default_files_switch=['id', 'os_family'],
+      default_files_switch=["id", "os_family"],
       indent_width=6,
-                       v1_path_prefix='') %}
+      use_subpath=False
+    ) %}
 {#-
     Returns a valid value for the "source" parameter of a "file.managed"
     state function. This makes easier the usage of the Template Override and
     Files Switch (TOFS) pattern.
-
     Params:
       * source_files: ordered list of files to look for
-      * lookup: key under '<tplroot>:tofs:source_files' to override
+      * lookup: key under "<tplroot>:tofs:source_files" to prepend to the
         list of source files
       * default_files_switch: if there's no config (e.g. pillar)
-        '<tplroot>:tofs:files_switch' this is the ordered list of grains to
+        "<tplroot>:tofs:files_switch" this is the ordered list of grains to
         use as selector switch of the directories under
         "<path_prefix>/files"
-      * indent_witdh: indentation of the result value to conform to YAML
+      * indent_width: indentation of the result value to conform to YAML
-      * v1_path_prefix: (deprecated) only used for injecting a path prefix into
+      * use_subpath: defaults to `False` but if set, lookup the source file
-        the source, to support older TOFS configs
+        recursively from the current state directory up to `tplroot`
-
     Example (based on a `tplroot` of `xxx`):
-
     If we have a state:
-
       Deploy configuration:
         file.managed:
           - name: /etc/yyy/zzz.conf
-          - source: {{ files_switch(['/etc/yyy/zzz.conf', '/etc/yyy/zzz.conf.jinja'],
+          - source: {{ files_switch(
-                                    lookup='Deploy configuration'
+                         ["/etc/yyy/zzz.conf", "/etc/yyy/zzz.conf.jinja"],
+                         lookup="Deploy configuration",
                        ) }}
           - template: jinja
-
     In a minion with id=theminion and os_family=RedHat, it's going to be
     rendered as:
-
       Deploy configuration:
         file.managed:
           - name: /etc/yyy/zzz.conf
@@ -48,54 +45,71 @@
           - template: jinja
 #}
 {#-   Get the `tplroot` from `tpldir` #}
-  {%- set tplroot = tpldir.split('/')[0] %}
+{%-   set tplroot = tpldir.split("/")[0] %}
-  {%- set path_prefix = salt['config.get'](tplroot ~ ':tofs:path_prefix', tplroot) %}
+{%-   set path_prefix = salt["config.get"](tplroot ~ ":tofs:path_prefix", tplroot) %}
-  {%- set files_dir = salt['config.get'](tplroot ~ ':tofs:dirs:files', 'files') %}
+{%-   set files_dir = salt["config.get"](tplroot ~ ":tofs:dirs:files", "files") %}
-  {%- set files_switch_list = salt['config.get'](
+{%-   set files_switch_list = salt["config.get"](
-      tplroot ~ ':tofs:files_switch',
+        tplroot ~ ":tofs:files_switch", default_files_switch
-      default_files_switch
       ) %}
-  {#- Lookup source_files (v2), files (v1), or fallback to source_files parameter #}
+{#-   Lookup source_files (v2), files (v1), or fallback to an empty list #}
-  {%- set src_files = salt['config.get'](
+{%-   set src_files = salt["config.get"](
-      tplroot ~ ':tofs:source_files:' ~ lookup,
+        tplroot ~ ":tofs:source_files:" ~ lookup,
-      salt['config.get'](
+        salt["config.get"](tplroot ~ ":tofs:files:" ~ lookup, []),
-          tplroot ~ ':tofs:files:' ~ lookup,
-          source_files
-      )
       ) %}
-  {#- Only add to [''] when supporting older TOFS implementations #}
+{#-   Append the default source_files #}
-  {%- set path_prefix_exts = [''] %}
+{%-   set src_files = src_files + source_files %}
-  {%- if v1_path_prefix != '' %}
+{#-   Only add to [""] when supporting older TOFS implementations #}
-    {%- do path_prefix_exts.append(v1_path_prefix) %}
+{%-   set path_prefix_exts = [""] %}
+{%-   if use_subpath and tplroot != tpldir %}
+{#-     Walk directory tree to find {{ files_dir }} #}
+{%-     set subpath_parts = tpldir.lstrip(tplroot).lstrip("/").split("/") %}
+{%-     for path in subpath_parts %}
+{%-       set subpath = subpath_parts[0 : loop.index] | join("/") %}
+{%-       do path_prefix_exts.append("/" ~ subpath) %}
+{%-     endfor %}
 {%-   endif %}
-  {%- for path_prefix_ext in path_prefix_exts %}
+{%-   for path_prefix_ext in path_prefix_exts | reverse %}
 {%-     set path_prefix_inc_ext = path_prefix ~ path_prefix_ext %}
 {#-     For older TOFS implementation, use `files_switch` from the config #}
 {#-     Use the default, new method otherwise #}
-    {%- set fsl = salt['config.get'](
+{%-     set fsl = salt["config.get"](
-        tplroot ~ path_prefix_ext|replace('/', ':') ~ ':files_switch',
+          tplroot ~ path_prefix_ext | replace("/", ":") ~ ":files_switch",
-        files_switch_list
+          files_switch_list,
         ) %}
 {#-     Append an empty value to evaluate as `default` in the loop below #}
-    {%- if '' not in fsl %}
+{%-     if "" not in fsl %}
-      {%- do fsl.append('') %}
+{%-       set fsl = fsl + [""] %}
 {%-     endif %}
 {%-     for fs in fsl %}
 {%-       for src_file in src_files %}
 {%-         if fs %}
-          {%- set fs_dir = salt['config.get'](fs, fs) %}
+{%-           set fs_dirs = salt["config.get"](fs, fs) %}
 {%-         else %}
-          {%- set fs_dir = salt['config.get'](tplroot ~ ':tofs:dirs:default', 'default') %}
+{%-           set fs_dirs = salt["config.get"](
+                tplroot ~ ":tofs:dirs:default", "default"
+              ) %}
 {%-         endif %}
-        {%- set url = [
+{#-         Force the `config.get` lookup result as a list where necessary #}
-            '- salt:/',
+{#-         since we need to also handle grains that are lists #}
-            path_prefix_inc_ext.strip('/'),
+{%-         if fs_dirs is string %}
-            files_dir.strip('/'),
+{%-           set fs_dirs = [fs_dirs] %}
-            fs_dir.strip('/'),
+{%-         endif %}
-            src_file.strip('/'),
+{%-         for fs_dir in fs_dirs %}
-        ] | select | join('/') %}
+{#-           strip empty elements by using a select #}
+{%-           set url = (
+                [
+                  "- salt:/",
+                  path_prefix_inc_ext.strip("/"),
+                  files_dir.strip("/"),
+                  fs_dir.strip("/"),
+                  src_file.strip("/"),
+                ]
+                | select
+                | join("/")
+              ) %}
 {{ url | indent(indent_width, true) }}
 {%-         endfor %}
 {%-       endfor %}
 {%-     endfor %}
+{%-   endfor %}
 {%- endmacro %}