From 6bb7e31c1d27ba06aeb8927f34f24a2f11dc4c48 Mon Sep 17 00:00:00 2001 From: "Sean P. McDonald" Date: Wed, 25 Sep 2024 12:30:02 -0500 Subject: [PATCH] (PE-39250) Remove project APIs and configuration Removes access to file_metadata and compile options which allowed fetching data and compiling for bolt projects. This commit does not remove all project based code, it is restricted to covering enough ground to remove access to any functionality, but there may still be some internal code left over. --- .../clj_kondo/slingshot/try_plus.clj | 44 +++ .../imports/clj-kondo/slingshot/config.edn | 2 + .../legacy_routes/legacy_routes_service.clj | 2 - .../services/master/file_serving.clj | 372 ------------------ .../services/master/master_core.clj | 75 +--- .../services/master/master_service.clj | 4 - .../puppetserver-lib/puppet/server/master.rb | 2 +- .../services/master/master_service_test.clj | 152 ------- .../services/master/file_serving_test.clj | 62 --- .../services/master/master_core_test.clj | 29 +- 10 files changed, 62 insertions(+), 682 deletions(-) create mode 100644 .clj-kondo/imports/clj-kondo/slingshot/clj_kondo/slingshot/try_plus.clj create mode 100644 .clj-kondo/imports/clj-kondo/slingshot/config.edn delete mode 100644 src/clj/puppetlabs/services/master/file_serving.clj delete mode 100644 test/unit/puppetlabs/services/master/file_serving_test.clj diff --git a/.clj-kondo/imports/clj-kondo/slingshot/clj_kondo/slingshot/try_plus.clj b/.clj-kondo/imports/clj-kondo/slingshot/clj_kondo/slingshot/try_plus.clj new file mode 100644 index 000000000..48a4c174e --- /dev/null +++ b/.clj-kondo/imports/clj-kondo/slingshot/clj_kondo/slingshot/try_plus.clj @@ -0,0 +1,44 @@ +(ns clj-kondo.slingshot.try-plus + (:require [clj-kondo.hooks-api :as api])) + +(defn expand-catch [catch-node] + (let [[catch catchee & exprs] (:children catch-node) + catchee-sexpr (api/sexpr catchee)] + (cond (vector? catchee-sexpr) + (let [[selector & exprs] exprs] + (api/list-node + [catch (api/token-node 'Exception) (api/token-node '_e#) + (api/list-node + (list* (api/token-node 'let) + (api/vector-node [selector (api/token-node nil)]) + exprs))])) + :else catch-node))) + +(defn try+ [{:keys [node]}] + (let [children (rest (:children node)) + [body catches] + (loop [body children + body-exprs [] + catches []] + (if (seq body) + (let [f (first body) + f-sexpr (api/sexpr f)] + (if (and (seq? f-sexpr) (= 'catch (first f-sexpr))) + (recur (rest body) + body-exprs + (conj catches (expand-catch f))) + (recur (rest body) + (conj body-exprs f) + catches))) + [body-exprs catches])) + new-node (api/list-node + [(api/token-node 'let) + (api/vector-node + [(api/token-node '&throw-context) (api/token-node nil)]) + (api/token-node '&throw-context) ;; use throw-context to avoid warning + (with-meta (api/list-node (list* (api/token-node 'try) + (concat body catches))) + (meta node))])] + ;; (prn (api/sexpr new-node)) + {:node new-node})) + diff --git a/.clj-kondo/imports/clj-kondo/slingshot/config.edn b/.clj-kondo/imports/clj-kondo/slingshot/config.edn new file mode 100644 index 000000000..446d4f0ca --- /dev/null +++ b/.clj-kondo/imports/clj-kondo/slingshot/config.edn @@ -0,0 +1,2 @@ +{:hooks + {:analyze-call {slingshot.slingshot/try+ clj-kondo.slingshot.try-plus/try+}}} diff --git a/src/clj/puppetlabs/services/legacy_routes/legacy_routes_service.clj b/src/clj/puppetlabs/services/legacy_routes/legacy_routes_service.clj index 270cffaf5..5a5dd3043 100644 --- a/src/clj/puppetlabs/services/legacy_routes/legacy_routes_service.clj +++ b/src/clj/puppetlabs/services/legacy_routes/legacy_routes_service.clj @@ -44,8 +44,6 @@ (constantly nil) false nil - nil - nil (get-in config [:puppetserver :certname]))) master-route-handler (comidi/routes->handler master-routes) master-mount (master-core/get-master-mount diff --git a/src/clj/puppetlabs/services/master/file_serving.clj b/src/clj/puppetlabs/services/master/file_serving.clj deleted file mode 100644 index b7c3c1aa3..000000000 --- a/src/clj/puppetlabs/services/master/file_serving.clj +++ /dev/null @@ -1,372 +0,0 @@ -(ns puppetlabs.services.master.file-serving - (:require [bidi.bidi :as bidi] - [clojure.java.io :as io] - [clojure.string :as str] - [digest :as digest] - [me.raynes.fs :as fs] - [puppetlabs.i18n.core :as i18n] - [puppetlabs.puppetserver.common :as common] - [puppetlabs.ring-middleware.utils :as middleware-utils] - [ring.util.response :as rr]) - (:import com.sun.security.auth.module.UnixSystem - [java.nio.file Files FileSystems FileVisitOption FileVisitResult LinkOption Paths SimpleFileVisitor])) - -(def follow-links - (into-array LinkOption [])) - -(defn as-path - [string & more] - (Paths/get string (into-array String more))) - -(defn path-exists? - [path] - (Files/exists path follow-links)) - -(defn get-checksum - [file algorithm] - (case algorithm - "md5" (str "{md5}" (digest/md5 (io/file file))) - "sha256" (str "{sha256}" (digest/sha-256 (io/file file))) - :default (throw (Exception. "Unsupported digest")))) - -(defn read-attributes - [path checksum-type ignore-source-permissions] - (let [attributes (Files/readAttributes path "unix:*" follow-links)] - {:owner (if ignore-source-permissions - (.getUid (UnixSystem.)) - (get attributes "uid")) - :group (if ignore-source-permissions - (.getGid (UnixSystem.)) - (get attributes "gid")) - :mode (if ignore-source-permissions - 0644 ;; Yes, Puppet even sends 644 for directories in this case - (bit-and (get attributes "mode") 07777)) - :type (if (get attributes "isDirectory") - "directory" - "file") - :checksum (if (get attributes "isDirectory") - {:type "ctime" - :value (str "{ctime}" (get attributes "ctime"))} - {:type checksum-type - :value (get-checksum path checksum-type)})})) - -(defn get-file-metadata - [checksum-type ignore-source-permissions [relative-path root]] - (merge (read-attributes (.resolve root relative-path) checksum-type ignore-source-permissions) - {:path (.toString root) - :relative_path (.toString relative-path) - :links "follow" - :destination nil})) - -(defn relativize - "A version of java's Path#relativize that matches puppet's behavior" - [root path] - (let [relative-path (.relativize root path)] - (if (= (.toString relative-path) "") - (as-path ".") - relative-path))) - -(defn make-visitor - [tree root ignores] - (let [filesystem (FileSystems/getDefault) - matchers (map #(.getPathMatcher filesystem (str "glob:" %)) ignores) - ignore? (fn [path] - (let [file-name (.getFileName path)] - (some #(.matches % file-name) matchers)))] - (proxy [SimpleFileVisitor] [] - (preVisitDirectory [path attributes] - (if (ignore? path) - FileVisitResult/SKIP_SUBTREE - (do - (swap! tree assoc (relativize root path) root) - FileVisitResult/CONTINUE))) - (visitFile [path attributes] - (when-not (ignore? path) - (swap! tree assoc (relativize root path) root)) - FileVisitResult/CONTINUE)))) - -(defn walk-directory - [path ignores] - (let [results (atom {}) - absolute-path (.toAbsolutePath path) - visitor (make-visitor results absolute-path ignores)] - (Files/walkFileTree absolute-path #{FileVisitOption/FOLLOW_LINKS} Integer/MAX_VALUE visitor) - @results)) - -(defn is-bolt-project? - [dir] - (or (fs/exists? (str dir "/bolt-project.yaml")) - (fs/exists? (str dir "/bolt.yaml")) - (fs/exists? (str dir "/Boltdir")))) - -(defn get-project-root - "Lookup a project by name and return the path where its project files are - located as a string." - [projects-dir versioned-project] - (let [boltdir-path (str projects-dir "/" versioned-project "/Boltdir")] - (if (fs/exists? boltdir-path) - boltdir-path - (str projects-dir "/" versioned-project)))) - -(defn list-dirs-in-paths - "Given a list of directories, return a list of all the subdirectories - concatenated, preserving the order of the original list. Directories that - don't exist will be ignored." - [paths] - (->> paths - (filter fs/exists?) - (mapcat (comp sort fs/list-dir)) - (filter fs/directory?))) - -(defn dirs-in-project-modulepath - "List all directories in a bolt project's modulepath. Returns a sequence of - File objects." - [modulepath project-root] - (->> modulepath - (map #(str project-root "/" %)) - list-dirs-in-paths)) - -(defn find-project-module - "Given the path of a project, the name of a module, and the modulepath defined - in the project's config (can be nil) search the project's module path and - return the path to that module as a File, or nil." - [bolt-builtin-content-dir project-root module modulepath] - (->> bolt-builtin-content-dir - list-dirs-in-paths - (concat (dirs-in-project-modulepath modulepath project-root)) - (filter #(= module (fs/base-name %))) - first)) - -(defn mount->path-component - "Map the fileserving \"mount\" to the subdirectory of modules it will serve - files out of." - [mount] - (case mount - "modules" "files" - ;; default to itself - mount)) - -(defn read-bolt-project-config - [project-dir] - (let [config-path (str project-dir "/bolt-project.yaml")] - (when (fs/file? config-path) - (common/parse-yaml (slurp config-path))))) - -(defn parse-modulepath - "The modulepath for a bolt project can either be an array of paths or a string - with a path separator." - [modulepath] - (if (string? modulepath) - (str/split modulepath #":") - modulepath)) - -(defn get-project-modulepath - "Given a map representing the bolt project configuration, return the - modulepath. If the modulepath is not defined or the configuration map passed - is nil then return the default modulepath." - [project-config] - ;; Note that the :modulepath config key's useful when the project - ;; organizes its modules inside relative directories. It doesn't - ;; make sense for absolute paths since those may not exist - ;; on the Puppetserver host. - (-> (get project-config :modulepath "modules") - (parse-modulepath) - (concat [".modules"]))) - -(defn find-project-file - "Find a file in a project using the parameters from a `file_content` request. - Returns the path as as string. If the module name is the same as the project - name then files are served from the project directly." - [bolt-builtin-content-dir bolt-projects-dir versioned-project mount module path] - (when (is-bolt-project? (str bolt-projects-dir "/" versioned-project)) - (let [project-root (get-project-root bolt-projects-dir versioned-project) - project-config (read-bolt-project-config project-root) - project-name (get project-config :name) - modulepath (get-project-modulepath project-config) - module-root (if (= project-name module) - project-root - (find-project-module bolt-builtin-content-dir project-root module modulepath)) - file-path (str module-root "/" (mount->path-component mount) "/" path)] - (when (fs/exists? file-path) - file-path)))) - -(defn mount-dirs-in-modulepath - "Collect all the paths represented by a specific mount that are found in the - modulepath. If `project-as-module?` is truthy then include any mount - directory found at the top level of the project." - [bolt-builtin-content-dir mount modulepath project-root project-as-module?] - (let [sub-dir (case mount - "plugins" "lib" - "pluginfacts" "facts.d") - modules-in-modulepath (concat - (dirs-in-project-modulepath modulepath project-root) - (list-dirs-in-paths bolt-builtin-content-dir)) - module-dirs (if project-as-module? - (cons (fs/file project-root) modules-in-modulepath) - modules-in-modulepath)] - (->> module-dirs - (map #(.resolve (.toPath %) sub-dir)) - (filter path-exists?)))) - -(defn project-configured-as-module? - "Given a project config, return whether it is configured to use the project - itself as a module. Users opt in to this by including a project name setting - in the configuration file. " - [project-config] - (boolean (get project-config :name))) - -(defn plugin-file-if-exists - "Given a relative path as received by the plugins mount, and the path of a lib - dir in a module, return the path of the file if it exists in the lib dir." - [relative-path lib-root] - (let [file-path (.resolve lib-root relative-path)] - (when (path-exists? file-path) - (.toString file-path)))) - -(defn find-project-plugin-file - "Given a relative path as received by the plugins mount, search all lib dirs - and return the path to the file if it's found in any of them." - [bolt-builtin-content-dir bolt-projects-dir versioned-project mount relative-path] - (let [project-root (get-project-root bolt-projects-dir versioned-project) - project-config (read-bolt-project-config project-root) - project-as-module? (project-configured-as-module? project-config) - modulepath (get-project-modulepath project-config) - mount-dirs (mount-dirs-in-modulepath bolt-builtin-content-dir mount modulepath project-root project-as-module?)] - (some (partial plugin-file-if-exists relative-path) mount-dirs))) - -(defn get-plugins-metadata - "Return the metadata for pluginsync. This scans the lib directories of all - modules and returns a list of files smashed together." - [bolt-builtin-content-dir bolt-projects-dir versioned-project mount checksum-type ignores ignore-source-permissions] - (when (is-bolt-project? (str bolt-projects-dir "/" versioned-project)) - (let [project-root (get-project-root bolt-projects-dir versioned-project) - project-config (read-bolt-project-config project-root) - project-as-module? (project-configured-as-module? project-config) - modulepath (get-project-modulepath project-config) - files (->> (mount-dirs-in-modulepath bolt-builtin-content-dir mount modulepath project-root project-as-module?) - (map #(walk-directory % ignores)) - reverse - (apply merge))] - (map (partial get-file-metadata checksum-type ignore-source-permissions) files)))) - -(def project-routes - "Bidi routing table for project file_content endpoint. This is done separately - because we have to do additional routing after branching on the query - parameter, which is not natively supported in bidi." - - ["" {[[#"modules|scripts|tasks" :mount-point] "/" :module "/" [#".+" :file-path]] :basic - [[#"plugins|pluginfacts" :mount-point] #"/?" [#".*" :file-path]] :pluginsync}]) - -(defn make-file-content-response - "Given a path to a file, generate an appropriate ring response map. Returns a - 404 response if passed `nil`." - [file requested-file-path] - (if file - (-> file - rr/file-response - (rr/content-type "application/octet-stream")) - {:status 404 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "Could not find file_content for path: {0}" requested-file-path)})) - -(defn handle-project-file-content - "Handle a file_content request for a bolt project." - [bolt-builtin-content-dir bolt-projects-dir request] - (let [versioned-project (get-in request [:params "versioned_project"]) - path (get-in request [:params :rest]) - match (bidi/match-route project-routes path)] - (when match - (let [mount-point (get-in match [:route-params :mount-point]) - mount-type (get-in match [:handler]) - module (get-in match [:route-params :module]) - file-path (get-in match [:route-params :file-path])] - (case mount-type - :basic (make-file-content-response - (find-project-file bolt-builtin-content-dir bolt-projects-dir versioned-project mount-point module file-path) - file-path) - :pluginsync (make-file-content-response - (find-project-plugin-file bolt-builtin-content-dir bolt-projects-dir versioned-project mount-point file-path) - file-path) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "Unsupported mount: {0}" mount-point)}))))) - -(defn file-content-handler - "Handle file_content requests and dispatch them to the correct handler for - environments or projects." - [bolt-builtin-content-dir bolt-projects-dir ruby-request-handler request] - (let [versioned-project (get-in request [:params "versioned_project"]) - environment (get-in request [:params "environment"])] - (cond - (and versioned-project environment) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "A file_content request cannot specify both `environment` and `versioned_project` query parameters.")} - (and (nil? versioned-project) (nil? environment)) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "A file_content request must include an `environment` or `versioned_project` query parameter.")} - versioned-project - (handle-project-file-content bolt-builtin-content-dir bolt-projects-dir request) - - :else - (ruby-request-handler request)))) - -(defn metadatas-params-errors - "Reject requests with parameter values that we haven't implemented yet. - Returns a list of errors." - [params] - (->> - (for [[value default] {"recurse" "true" - "links" "follow" - "source_permissions" "ignore" - "recurselimit" "infinite"}] - (when (and (get params value) - (not (= value default))) - (str "The only supported value of `" value "` at this time is `" default "`"))) - (filter (comp not nil?)))) - -(defn handle-project-file-metadatas - "Handle a file_metadatas request for a bolt project." - [bolt-builtin-content-dir bolt-projects-dir request] - (let [versioned-project (get-in request [:params "versioned_project"]) - path (get-in request [:params :rest]) - match (bidi/match-route project-routes path)] - (when match - (let [mount-point (get-in match [:route-params :mount-point]) - mount-type (get-in match [:handler]) - ignore (get-in request [:params "ignore"] []) - checksum-type (get-in request [:params "checksum_type"] "sha256")] - (case mount-type - :pluginsync (let [errors (metadatas-params-errors (get-in request [:params]))] - (if (empty? errors) - (middleware-utils/json-response - 200 - (get-plugins-metadata bolt-builtin-content-dir bolt-projects-dir versioned-project mount-point checksum-type ignore true)) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (str/join "\n" (cons "Not all parameter values are supported in this implementation: " errors))})) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "Unsupported mount: {0}" mount-point)}))))) - -(defn file-metadatas-handler - "Handle file_metadatas requests and dispatch them to the correct handler for - environments or projects." - [bolt-builtin-content-dir bolt-projects-dir ruby-request-handler request] - (let [versioned-project (get-in request [:params "versioned_project"]) - environment (get-in request [:params "environment"])] - (cond - (and versioned-project environment) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "A file_metadatas request cannot specify both `environment` and `versioned_project` query parameters.")} - (and (nil? versioned-project) (nil? environment)) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "A file_metadatas request must include an `environment` or `versioned_project` query parameter.")} - versioned-project - (handle-project-file-metadatas bolt-builtin-content-dir bolt-projects-dir request) - - :else - (ruby-request-handler request)))) diff --git a/src/clj/puppetlabs/services/master/master_core.clj b/src/clj/puppetlabs/services/master/master_core.clj index 5dfea3ae6..981d59cda 100644 --- a/src/clj/puppetlabs/services/master/master_core.clj +++ b/src/clj/puppetlabs/services/master/master_core.clj @@ -18,7 +18,6 @@ [puppetlabs.puppetserver.ringutils :as ringutils] [puppetlabs.ring-middleware.core :as middleware] [puppetlabs.ring-middleware.utils :as middleware-utils] - [puppetlabs.services.master.file-serving :as file-serving] [puppetlabs.services.protocols.jruby-puppet :as jruby-protocol] [puppetlabs.trapperkeeper.services.status.status-core :as status-core] [ring.middleware.params :as ring] @@ -963,55 +962,26 @@ (catch Throwable e (log/info e (i18n/trs "Failed to report action"))))))))) -(defn parse-project-compile-data - "Parse data required to compile a catalog inside a project. Data required includes - * Root path to project - * modulepath - * hiera config - * project_name" - [request-options - versioned-project - bolt-projects-dir] - (let [project-root (file-serving/get-project-root bolt-projects-dir versioned-project) - project-config (file-serving/read-bolt-project-config project-root)] - (assoc request-options "project_root" project-root - "modulepath" (map #(str project-root "/" %) (file-serving/get-project-modulepath project-config)) - "hiera_config" (str project-root "/" (get project-config :hiera-config "hiera.yaml")) - "project_name" (:name project-config)))) - (schema/defn ^:always-validate compile-fn :- IFn [jruby-service :- (schema/protocol jruby-protocol/JRubyPuppetService) current-code-id-fn :- IFn - boltlib-path :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str)] + boltlib-path :- (schema/maybe [schema/Str])] (fn [request] (let [request-options (-> request :body slurp (validated-body CompileRequest)) - versioned-project (get request-options "versioned_project") environment (get request-options "environment")] - ;; Check to ensure environment/versioned_project are mutually exlusive and - ;; at least one of them is set. - (cond - (and versioned-project environment) - {:status 400 - :headers {"Content-Type" "text/plain"} - :body (i18n/tru "A compile request cannot specify both `environment` and `versioned_project` parameters.")} - - (and (nil? versioned-project) (nil? environment)) + ;; Check to ensure environment is set. + + (if (nil? environment) {:status 400 :headers {"Content-Type" "text/plain"} :body (i18n/tru "A compile request must include an `environment` or `versioned_project` parameter.")} - :else - (let [compile-options (if versioned-project - ;; we need to parse some data from the project config for project compiles - (parse-project-compile-data request-options versioned-project bolt-projects-dir) - ;; environment compiles only need to set the code ID - (assoc request-options + (let [compile-options (assoc request-options "code_id" - (current-code-id-fn environment)))] + (current-code-id-fn environment))] {:status 200 :headers {"Content-Type" "application/json"} :body (json/encode @@ -1035,9 +1005,8 @@ [jruby-service :- (schema/protocol jruby-protocol/JRubyPuppetService) wrap-with-jruby-queue-limit :- IFn current-code-id-fn :- IFn - boltlib-path :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str)] - (-> (compile-fn jruby-service current-code-id-fn boltlib-path bolt-projects-dir) + boltlib-path :- (schema/maybe [schema/Str])] + (-> (compile-fn jruby-service current-code-id-fn boltlib-path) (jruby-request/wrap-with-jruby-instance jruby-service) wrap-with-jruby-queue-limit jruby-request/wrap-with-error-handling)) @@ -1061,17 +1030,14 @@ v3-ruby-routes :- bidi-schema/RoutePair "v3 route tree for the ruby side of the master service." [request-handler :- IFn - bolt-builtin-content-dir :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str) certname :- schema/Str] (comidi/routes (comidi/GET ["/node/" [#".*" :rest]] request (request-handler request)) (comidi/GET ["/file_content/" [#".*" :rest]] request - ;; Not strictly ruby routes anymore because of this - (file-serving/file-content-handler bolt-builtin-content-dir bolt-projects-dir request-handler (ring/params-request request))) + (request-handler request)) (comidi/GET ["/file_metadatas/" [#".*" :rest]] request - (file-serving/file-metadatas-handler bolt-builtin-content-dir bolt-projects-dir request-handler (ring/params-request request))) + (request-handler request)) (comidi/GET ["/file_metadata/" [#".*" :rest]] request (request-handler request)) (comidi/GET ["/file_bucket_file/" [#".*" :rest]] request @@ -1101,8 +1067,7 @@ current-code-id-fn :- IFn cache-enabled :- schema/Bool wrap-with-jruby-queue-limit :- IFn - boltlib-path :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str)] + boltlib-path :- (schema/maybe [schema/Str])] (let [class-handler (create-cacheable-info-handler-with-middleware (fn [jruby env] (some-> jruby-service @@ -1137,8 +1102,7 @@ jruby-service wrap-with-jruby-queue-limit current-code-id-fn - boltlib-path - bolt-projects-dir)] + boltlib-path)] (comidi/routes (comidi/POST "/compile" request (compile-handler' request)) @@ -1190,19 +1154,16 @@ environment-class-cache-enabled :- schema/Bool wrap-with-jruby-queue-limit :- IFn boltlib-path :- (schema/maybe [schema/Str]) - bolt-builtin-content-dir :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str) certname :- schema/Str] (comidi/context "/v3" - (v3-ruby-routes ruby-request-handler bolt-builtin-content-dir bolt-projects-dir certname) + (v3-ruby-routes ruby-request-handler certname) (comidi/wrap-routes (v3-clojure-routes jruby-service get-code-content-fn current-code-id-fn environment-class-cache-enabled wrap-with-jruby-queue-limit - boltlib-path - bolt-projects-dir) + boltlib-path) clojure-request-wrapper))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1305,8 +1266,6 @@ current-code-id-fn :- IFn environment-class-cache-enabled :- schema/Bool boltlib-path :- (schema/maybe [schema/Str]) - bolt-builtin-content-dir :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str) certname :- schema/Str] (comidi/routes (v3-routes ruby-request-handler @@ -1317,8 +1276,6 @@ environment-class-cache-enabled wrap-with-jruby-queue-limit boltlib-path - bolt-builtin-content-dir - bolt-projects-dir certname) (v4-routes clojure-request-wrapper jruby-service @@ -1381,8 +1338,6 @@ wrap-with-jruby-queue-limit :- IFn environment-class-cache-enabled :- schema/Bool boltlib-path :- (schema/maybe [schema/Str]) - bolt-builtin-content-dir :- (schema/maybe [schema/Str]) - bolt-projects-dir :- (schema/maybe schema/Str) certname :- schema/Str] (let [ruby-request-handler (wrap-middleware handle-request wrap-with-authorization-check @@ -1400,8 +1355,6 @@ current-code-id environment-class-cache-enabled boltlib-path - bolt-builtin-content-dir - bolt-projects-dir certname))) (def MasterStatusV1 diff --git a/src/clj/puppetlabs/services/master/master_service.clj b/src/clj/puppetlabs/services/master/master_service.clj index 1c332f100..e97563fc3 100644 --- a/src/clj/puppetlabs/services/master/master_service.clj +++ b/src/clj/puppetlabs/services/master/master_service.clj @@ -115,8 +115,6 @@ certname (get-in config [:puppetserver :certname]) localcacert (get-in config [:puppetserver :localcacert]) puppet-version (get-in config [:puppetserver :puppet-version]) - bolt-builtin-content-dir (get-in config [:bolt :builtin-content-dir] []) - bolt-projects-dir (get-in config [:bolt :projects-dir]) max-queued-requests (get-in config [:jruby-puppet :max-queued-requests] 0) max-retry-delay (get-in config [:jruby-puppet :max-retry-delay] 1800) settings (ca/config->master-settings config) @@ -147,8 +145,6 @@ wrap-with-jruby-queue-limit environment-class-cache-enabled boltlib-path - bolt-builtin-content-dir - bolt-projects-dir certname)) routes (comidi/context path ring-app) route-metadata (comidi/route-metadata routes) diff --git a/src/ruby/puppetserver-lib/puppet/server/master.rb b/src/ruby/puppetserver-lib/puppet/server/master.rb index 9414b7ae9..d9a18188a 100644 --- a/src/ruby/puppetserver-lib/puppet/server/master.rb +++ b/src/ruby/puppetserver-lib/puppet/server/master.rb @@ -248,7 +248,7 @@ def check_cadir_for_deprecation_warning # Each array element is examined, if it is expected to be a map # we call back to the convert_java_args_to_ruby method, if it # is expected to be an array, we recurse otherwise we do not modify - # the value. + # the value. def resolve_java_objects_from_list(list) list.map do |value| if value.kind_of?(Java::ClojureLang::IPersistentMap) diff --git a/test/integration/puppetlabs/services/master/master_service_test.clj b/test/integration/puppetlabs/services/master/master_service_test.clj index 257e9c1c9..b45f50ce1 100644 --- a/test/integration/puppetlabs/services/master/master_service_test.clj +++ b/test/integration/puppetlabs/services/master/master_service_test.clj @@ -770,155 +770,3 @@ Integer/parseInt)] (is (= 503 status-code)) (is (<= 0 retry-after 1800)))))))) - -(deftest ^:integration project-file-content - (bootstrap-testutils/with-puppetserver-running - app - {:bolt {:builtin-content-dir ["./dev-resources/puppetlabs/services/master/master_core_test/builtin_bolt_content"] - :projects-dir "./dev-resources/puppetlabs/services/master/master_core_test/bolt_projects"} - :jruby-puppet {:gem-path gem-path - :max-active-instances 1 - :server-code-dir test-resources-code-dir - :server-conf-dir master-service-test-runtime-dir}} - - (testing "can retrieve file_content from the modules mount from a project" - (let [response (http-get "/puppet/v3/file_content/modules/utilities/etc/greeting?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "Good morning\n" (:body response))) - (is (= "13" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - ;; this also tests that we can retrieve file content from the .modules mount of the project - ;; since that is where this task is located - (testing "can retrieve file_content from the tasks mount from a project" - (let [response (http-get "/puppet/v3/file_content/tasks/utilities/blah?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "bye" (:body response))) - (is (= "3" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve file_content from the top level of a project" - (let [response (http-get "/puppet/v3/file_content/tasks/local/init.sh?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= ". $PT__installdir/helpers/files/marco.sh\nmarco\n" (:body response))) - (is (= "63" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve file_content from a project with an embedded structure" - (let [response (http-get "/puppet/v3/file_content/modules/test/packages?versioned_project=embedded_e19e09")] - (is (= 200 (:status response))) - (is (= "vim" (:body response))) - (is (= "3" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve file_content from a custom modulepath" - (let [response (http-get "/puppet/v3/file_content/modules/helpers/marco.sh?versioned_project=local_afafaf")] - (is (= 200 (:status response))) - (is (= "marco () {\n echo \"polo\"\n}\n" (:body response))) - (is (= "27" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve built-in file_content" - (let [response (http-get "/puppet/v3/file_content/tasks/bic_module_one/init.sh?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= ". $PT__installdir/helpers/files/marco.sh\nmarco\n" (:body response))) - (is (= "63" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve overriden built-in file_content" - (let [response (http-get "/puppet/v3/file_content/tasks/bic_module_one/init.sh?versioned_project=override_builtin_content")] - (is (= 200 (:status response))) - (is (= ". $PT__installdir/helpers/files/marco.sh\noverride_marco\n" (:body response))) - (is (= "73" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "cannot retrieve file_content from the default modulepath when a custom modulepath is set" - (let [response (http-get "/puppet/v3/file_content/tasks/utilities/blah?versioned_project=local_afafaf")] - (is (= 404 (:status response))))) - - (testing "mount point not found" - (let [response (http-get "/puppet/v3/file_content/glorb/test/packages?versioned_project=local_23")] - (is (= 404 (:status response))))) - - (testing "project not found" - (let [response (http-get "/puppet/v3/file_content/modules/test/packages?versioned_project=nothere")] - (is (= 404 (:status response))))) - - (testing "module not found" - (let [response (http-get "/puppet/v3/file_content/modules/nomodule/packages?versioned_project=embedded_e19e09")] - (is (= 404 (:status response))))) - - (testing "missing path?" - (let [response (http-get "/puppet/v3/file_content/modules/test/?versioned_project=embedded_e19e09")] - (is (= 404 (:status response))))) - - (testing "can retrieve plugin metadata" - (let [response (http-get "/puppet/v3/file_metadatas/plugins?versioned_project=local_23") - [file-entry] (filter #(= "puppet/monkey_patch.rb" (get % "relative_path")) (json/decode (:body response)))] - ;; Only check some of the entries that won't vary based on the test environment - (is (= nil (get file-entry "destination"))) - (is (= "file" (get file-entry "type"))) - (is (= "sha256" (get-in file-entry ["checksum" "type"]))) - (is (= "{sha256}76b2e03b82880885385595045033c4e3122e373c7023e037461a650ec85829ad" (get-in file-entry ["checksum" "value"]))) - ;; Does it choose from the right module? - (is (str/ends-with? (get file-entry "path") "modules/helpers/lib")))) - - (testing "can retrieve builtin plugin metadata" - (let [response (http-get "/puppet/v3/file_metadatas/plugins?versioned_project=local_23") - [file-entry] (filter #(= "puppet/builtin_monkey_patch.rb" (get % "relative_path")) (json/decode (:body response)))] - ;; Only check some of the entries that won't vary based on the test environment - (is (= nil (get file-entry "destination"))) - (is (= "file" (get file-entry "type"))) - (is (= "sha256" (get-in file-entry ["checksum" "type"]))) - (is (= "{sha256}76b2e03b82880885385595045033c4e3122e373c7023e037461a650ec85829ad" (get-in file-entry ["checksum" "value"]))) - ;; Does it choose from the right module? - (is (str/ends-with? (get file-entry "path") "bic_module_one/lib")))) - - (testing "can retrieve plugin files" - (let [response (http-get "/puppet/v3/file_content/plugins/puppet/monkey_patch.rb?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "class NilClass\n def empty?\n true\n end\nend\n" (:body response))) - (is (= "59" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve builtin plugin files" - (let [response (http-get "/puppet/v3/file_content/plugins/puppet/builtin_monkey_patch.rb?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "class NilClass\n def empty?\n true\n end\nend\n" (:body response))) - (is (= "59" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve overridden builtin plugin files" - (let [response (http-get "/puppet/v3/file_content/plugins/puppet/builtin_monkey_patch.rb?versioned_project=override_builtin_content")] - (is (= 200 (:status response))) - (is (= "overridden_class NilClass\n def empty?\n true\n end\nend\n" (:body response))) - (is (= "70" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve plugin files from the top level project lib dir" - (let [response (http-get "/puppet/v3/file_content/plugins/puppet/comment.rb?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "# This is the project\n" (:body response))) - (is (= "22" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "can retrieve pluginfacts metadata" - (let [response (http-get "/puppet/v3/file_metadatas/pluginfacts?versioned_project=local_23") - [file-entry] (filter #(= "something" (get % "relative_path")) (json/decode (:body response)))] - (is (= nil (get file-entry "destination"))) - (is (= "file" (get file-entry "type"))) - (is (= "sha256" (get-in file-entry ["checksum" "type"]))) - (is (= "{sha256}4bc453b53cb3d914b45f4b250294236adba2c0e09ff6f03793949e7e39fd4cc1" (get-in file-entry ["checksum" "value"]))))) - - (testing "can retrieve pluginfacts files" - (let [response (http-get "/puppet/v3/file_content/pluginfacts/unhelpful?versioned_project=local_23")] - (is (= 200 (:status response))) - (is (= "factually unhelpful\n" (:body response))) - (is (= "20" (get-in response [:headers "content-length"]))) - (is (= "application/octet-stream" (get-in response [:headers "content-type"]))))) - - (testing "doesn't support nonstandard options" - (let [response (http-get "/puppet/v3/file_metadatas/plugins?versioned_project=local_23&links=manage")] - (is (= 400 (:status response))) - (is (= "Not all parameter values are supported in this implementation: \nThe only supported value of `links` at this time is `follow`" - (:body response))))))) diff --git a/test/unit/puppetlabs/services/master/file_serving_test.clj b/test/unit/puppetlabs/services/master/file_serving_test.clj deleted file mode 100644 index c18a3b099..000000000 --- a/test/unit/puppetlabs/services/master/file_serving_test.clj +++ /dev/null @@ -1,62 +0,0 @@ -(ns puppetlabs.services.master.file-serving-test - (:require [clojure.test :refer [deftest is testing]] - [me.raynes.fs :as fs] - [puppetlabs.services.master.file-serving :refer [get-project-root - find-project-file - get-project-modulepath - read-attributes]]) - (:import java.nio.file.Paths)) - -(deftest bolt-projects-test - (let [bolt-builtin-content-dir ["dev-resources/puppetlabs/services/master/master_core_test/builtin_bolt_content"] - bolt-projects-dir "./dev-resources/puppetlabs/services/master/master_core_test/bolt_projects"] - (testing "find the root of a bolt project" - (testing "regular project" - (is (= (str bolt-projects-dir "/local_23") - (get-project-root bolt-projects-dir "local_23")))) - - (testing "embedded_e19e09 project (Boltdir)" - (is (= (str bolt-projects-dir "/embedded_e19e09/Boltdir") - (get-project-root bolt-projects-dir "embedded_e19e09"))))) - - (testing "finding a file in a project" - (testing "in the modules mount" - (is (= (fs/file (str bolt-projects-dir "/local_23/modules/helpers/files/marco.sh" )) - (fs/file (find-project-file bolt-builtin-content-dir bolt-projects-dir "local_23" "modules" "helpers" "marco.sh")))) - (is (= (fs/file (str bolt-projects-dir "/local_23/.modules/utilities/files/etc/greeting" )) - (fs/file (find-project-file bolt-builtin-content-dir bolt-projects-dir "local_23" "modules" "utilities" "etc/greeting"))))) - - (testing "finding a script in a project" - (testing "in the scripts mount" - (is (= (fs/file (str bolt-projects-dir "/local_23/modules/helpers/scripts/yogurt.sh" )) - (fs/file (find-project-file bolt-builtin-content-dir bolt-projects-dir "local_23" "scripts" "helpers" "yogurt.sh")))))) - - (testing "returns nil when a component is not found" - (is (= nil - (find-project-file bolt-builtin-content-dir bolt-projects-dir "fake" "modules" "helpers" "marco.sh"))) - (is (= nil - (find-project-file bolt-builtin-content-dir bolt-projects-dir "local_23" "fake" "helpers" "marco.sh"))) - (is (= nil - (find-project-file bolt-builtin-content-dir bolt-projects-dir "local_23" "modules" "fake" "marco.sh"))) - (is (= nil - (find-project-file bolt-builtin-content-dir bolt-projects-dir "local_23" "modules" "helpers" "fake"))) - (is (= nil - (find-project-file bolt-builtin-content-dir nil "local_23" "modules" "helpers" "marco.sh"))))))) - -(deftest get-project-modulepath-test - (testing "it returns the default modulepath when none is supplied" - (is (= ["modules" ".modules"] - (get-project-modulepath {})))) - - (testing "if a modulepath is supplied, then it returns modulepath + '.modules'" - (let [modulepath ["dir" "otherdir"]] - (is (= ["dir" "otherdir" ".modules"] - (get-project-modulepath {:modulepath modulepath})))))) - -(deftest metadata - (let [path (Paths/get - "./dev-resources/puppetlabs/services/master/master_core_test/bolt_projects/local_23/modules/helpers/files/marco.sh" - (into-array String [])) - attributes (read-attributes path "md5" true)] - (testing "just get the metadata" - (is (= "file" (:type attributes)))))) diff --git a/test/unit/puppetlabs/services/master/master_core_test.clj b/test/unit/puppetlabs/services/master/master_core_test.clj index 9386820c0..7b33895bc 100644 --- a/test/unit/puppetlabs/services/master/master_core_test.clj +++ b/test/unit/puppetlabs/services/master/master_core_test.clj @@ -50,8 +50,6 @@ (constantly nil) true nil - ["./dev-resources/puppetlabs/services/master/master_core_test/builtin_bolt_content"] - "./dev-resources/puppetlabs/services/master/master_core_test/bolt_projects" "test-certname") (comidi/routes->handler) (wrap-middleware identity puppet-version))) @@ -447,36 +445,11 @@ :trusted_facts {:values {}} :variables {:values {}}}))))] (is (= 200 (:status response))))) - (testing "compile endpoint for projects" + (testing "compile endpoint fails with no environment" (let [response (app (-> {:request-method :post :uri "/v3/compile" :content-type "application/json"} (ring-mock/body (json/encode {:certname "foo" - :versioned_project "fake_project" - :code_ast "{\"__pcore_something\": \"Foo\"}" - :facts {:values {}} - :trusted_facts {:values {}} - :variables {:values {}} - :options {:compile_for_plan true}}))))] - (is (= 200 (:status response))))) - (testing "compile endpoint fails with no environment or versioned_project" - (let [response (app (-> {:request-method :post - :uri "/v3/compile" - :content-type "application/json"} - (ring-mock/body (json/encode {:certname "foo" - :code_ast "{\"__pcore_something\": \"Foo\"}" - :facts {:values {}} - :trusted_facts {:values {}} - :variables {:values {}} - :options {:compile_for_plan true}}))))] - (is (= 400 (:status response))))) - (testing "compile endpoint fails with both environment and versioned_project" - (let [response (app (-> {:request-method :post - :uri "/v3/compile" - :content-type "application/json"} - (ring-mock/body (json/encode {:certname "foo" - :environment "production" - :versioned_project "fake_project" :code_ast "{\"__pcore_something\": \"Foo\"}" :facts {:values {}} :trusted_facts {:values {}}