From 8b2bf3953e7fd4d95af6233b98aeaa45c1d32e68 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Wed, 26 Apr 2023 13:06:00 -0400 Subject: [PATCH] docs: Update plugin documentation. (#59) --- .github/workflows/docs-pr.yml | 30 ++ cmd/readme_config_includer/generator.go | 264 ++++++++++++++++++ .../plugins/exporters/filewriter/README.md | 22 ++ .../filewriter/file_exporter_config.go | 1 + .../plugins/exporters/filewriter/sample.yaml | 1 - .../plugins/exporters/postgresql/README.md | 44 +++ .../postgresql/postgresql_exporter_config.go | 1 + conduit/plugins/importers/algod/README.md | 45 +++ .../importers/algod/algod_importer_config.go | 1 + conduit/plugins/importers/algod/sample.yaml | 8 +- .../plugins/importers/filereader/README.md | 26 ++ .../importers/filereader/filereader_config.go | 1 + .../processors/filterprocessor/README.md | 48 +++- .../processors/filterprocessor/config.go | 1 + docs/Configuration.md | 2 +- docs/GettingStarted.md | 4 - docs/plugins/algod.md | 20 -- docs/plugins/file_writer.md | 20 -- docs/plugins/filter_processor.md | 67 ----- docs/plugins/noop_exporter.md | 11 - docs/plugins/noop_processor.md | 11 - docs/plugins/postgresql.md | 23 -- docs/tutorials/IndexerMigration.md | 2 +- docs/tutorials/WritingBlocksToFile.md | 4 +- go.mod | 1 + go.sum | 2 + 26 files changed, 480 insertions(+), 180 deletions(-) create mode 100644 cmd/readme_config_includer/generator.go create mode 100644 conduit/plugins/exporters/filewriter/README.md create mode 100644 conduit/plugins/exporters/postgresql/README.md create mode 100644 conduit/plugins/importers/algod/README.md create mode 100644 conduit/plugins/importers/filereader/README.md rename docs/tutorials/FilterDeepDive.md => conduit/plugins/processors/filterprocessor/README.md (79%) delete mode 100644 docs/plugins/algod.md delete mode 100644 docs/plugins/file_writer.md delete mode 100644 docs/plugins/filter_processor.md delete mode 100644 docs/plugins/noop_exporter.md delete mode 100644 docs/plugins/noop_processor.md delete mode 100644 docs/plugins/postgresql.md diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml index aa5d636f..8eb85164 100644 --- a/.github/workflows/docs-pr.yml +++ b/.github/workflows/docs-pr.yml @@ -39,6 +39,36 @@ jobs: # Update hand-written documentation rm -rf docs/docs/get-details/conduit/ cp -r conduit/docs docs/docs/get-details/conduit + + # Copy the plugin docs using this ChatGPT special. + # Set the directory to search + search_dir="conduit/plugins" + target_base_dir=docs/docs/get-details/conduit/plugins + + # Loop through each README.md file found + find "$search_dir" -name "README.md" | while read readme_file; do + # Get the directory path of the README file + dir_path=$(dirname "$readme_file") + + # Get the path relative to search_dir + relative_path=${dir_path#$search_dir/} + + # Remove the last directory name from relative_path + output_dir=${relative_path%/*} + + # Create the output directory path + output_dir="$target_base_dir/$output_dir" + + # Create the output file path + output_file="$output_dir/$(basename "$dir_path" .md).md" + + # Create the output directory if it doesn't exist + mkdir -p "$output_dir" + + # Copy the README file to the output file path + cp "$readme_file" "$output_file" + done + # TODO: check if there are any changes before creating a PR? - name: Create algorand/docs Pull Request uses: peter-evans/create-pull-request@v4 diff --git a/cmd/readme_config_includer/generator.go b/cmd/readme_config_includer/generator.go new file mode 100644 index 00000000..db151366 --- /dev/null +++ b/cmd/readme_config_includer/generator.go @@ -0,0 +1,264 @@ +// This comes from the InfluxData, it is MIT licensed. +// +// This is a tool to embed configuration files into the README.md of all plugins +// It searches for YML sections in the plugins' README.md and detects includes specified in the form +// +// ```yml [@includeA.conf[ @includeB[ @...]] +// Whatever is in here gets replaced. +// ``` +// +// Then it will replace everything in this section by the concatenation of the file `includeA.conf`, `includeB` etc. +// content. The tool is not stateful, so it can be run multiple time with a stable result as long +// as the included files do not change. +package main + +import ( + "bytes" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/text" +) + +var ( + // Finds all comment section parts `<-- @includefile -->` + commentIncludesEx = regexp.MustCompile(``) + // Finds all YML sections of the form `yml @includefile` + ymlIncludesEx = regexp.MustCompile(`[\s"]+(@.+)+"?`) + // Extracts the `includefile` part + includeMatch = regexp.MustCompile(`(?:@([^\s"]+))+`) +) + +type includeBlock struct { + Includes []string + Start int + Stop int + Newlines bool +} + +func extractIncludeBlock(txt []byte, includesEx *regexp.Regexp, root string) *includeBlock { + includes := includesEx.FindSubmatch(txt) + if len(includes) != 2 { + return nil + } + block := includeBlock{} + for _, inc := range includeMatch.FindAllSubmatch(includes[1], -1) { + if len(inc) != 2 { + continue + } + include := filepath.FromSlash(string(inc[1])) + // Make absolute paths relative to the include-root if any + // Check original value to avoid platform specific slashes + if filepath.IsAbs(string(inc[1])) { + if root == "" { + log.Printf("Ignoring absolute include %q without include root...", include) + continue + } + include = filepath.Join(root, include) + } + include, err := filepath.Abs(include) + if err != nil { + log.Printf("Cannot resolve include %q...", include) + continue + } + if fi, err := os.Stat(include); err != nil || !fi.Mode().IsRegular() { + log.Printf("Ignoring include %q as it cannot be found or is not a regular file...", include) + continue + } + block.Includes = append(block.Includes, include) + } + return &block +} + +func insertInclude(buf *bytes.Buffer, include string) error { + file, err := os.Open(include) + if err != nil { + return fmt.Errorf("opening include %q failed: %w", include, err) + } + defer file.Close() + + // Write the include and make sure we get a newline + if _, err := io.Copy(buf, file); err != nil { + return fmt.Errorf("inserting include %q failed: %w", include, err) + } + return nil +} + +func insertIncludes(buf *bytes.Buffer, b *includeBlock) error { + // Insert newlines before and after + if b.Newlines { + if _, err := buf.Write([]byte("\n")); err != nil { + return errors.New("adding newline failed") + } + } + + // Insert all includes in the order they occurred + for _, include := range b.Includes { + if err := insertInclude(buf, include); err != nil { + return err + } + } + // Make sure we add a trailing newline + if !bytes.HasSuffix(buf.Bytes(), []byte("\n")) { + if _, err := buf.Write([]byte("\n")); err != nil { + return errors.New("adding newline failed") + } + } + + // Insert newlines before and after + if b.Newlines { + if _, err := buf.Write([]byte("\n")); err != nil { + return errors.New("adding newline failed") + } + } + + return nil +} + +func main() { + // Estimate Telegraf root to be able to handle absolute paths + cwd, err := os.Getwd() + if err != nil { + log.Fatalf("Cannot get working directory: %v", err) + } + cwd, err = filepath.Abs(cwd) + if err != nil { + log.Fatalf("Cannot resolve working directory: %v", err) + } + + var includeRoot string + if idx := strings.LastIndex(cwd, filepath.FromSlash("/plugins/")); idx > 0 { + includeRoot = cwd[:idx] + } + + // Get the file permission of the README for later use + inputFilename := "README.md" + inputFileInfo, err := os.Lstat(inputFilename) + if err != nil { + log.Fatalf("Cannot get file permissions: %v", err) + } + perm := inputFileInfo.Mode().Perm() + + // Read and parse the README markdown file + readme, err := os.ReadFile(inputFilename) + if err != nil { + log.Fatalf("Reading README failed: %v", err) + } + parser := goldmark.DefaultParser() + root := parser.Parse(text.NewReader(readme)) + + // Walk the markdown to identify the (YML) parts to replace + blocksToReplace := make([]*includeBlock, 0) + for rawnode := root.FirstChild(); rawnode != nil; rawnode = rawnode.NextSibling() { + // Only match YML code nodes + var txt []byte + var start, stop int + var newlines bool + var re *regexp.Regexp + switch node := rawnode.(type) { + case *ast.FencedCodeBlock: + if string(node.Language(readme)) != "yml" { + // Ignore any other node type or language + continue + } + // Extract the block borders + start = node.Info.Segment.Stop + 1 + stop = start + lines := node.Lines() + if lines.Len() > 0 { + stop = lines.At(lines.Len() - 1).Stop + } + txt = node.Info.Text(readme) + re = ymlIncludesEx + case *ast.Heading: + if node.ChildCount() < 2 { + continue + } + child, ok := node.LastChild().(*ast.RawHTML) + if !ok || child.Segments.Len() == 0 { + continue + } + segment := child.Segments.At(0) + if !commentIncludesEx.Match(segment.Value(readme)) { + continue + } + start = segment.Stop + 1 + stop = len(readme) // necessary for cases with no more headings + for rawnode = rawnode.NextSibling(); rawnode != nil; rawnode = rawnode.NextSibling() { + if h, ok := rawnode.(*ast.Heading); ok && h.Level <= node.Level { + if rawnode.Lines().Len() > 0 { + stop = rawnode.Lines().At(0).Start - h.Level - 1 + } else { + log.Printf("heading without lines: %s", string(rawnode.Text(readme))) + stop = start // safety measure to prevent removing all text + } + break + } + } + txt = segment.Value(readme) + re = commentIncludesEx + newlines = true + default: + // Ignore everything else + continue + } + + // Extract the includes from the node + block := extractIncludeBlock(txt, re, includeRoot) + if block != nil { + block.Start = start + block.Stop = stop + block.Newlines = newlines + blocksToReplace = append(blocksToReplace, block) + } + + // Catch the case of heading-end-search exhausted all nodes + if rawnode == nil { + break + } + } + + // Replace the content of the YML blocks with includes + var output bytes.Buffer + output.Grow(len(readme)) + offset := 0 + for _, b := range blocksToReplace { + // Copy everything up to the beginning of the block we want to replace and make sure we get a newline + if _, err := output.Write(readme[offset:b.Start]); err != nil { + log.Fatalf("Writing non-replaced content failed: %v", err) + } + if !bytes.HasSuffix(output.Bytes(), []byte("\n")) { + if _, err := output.Write([]byte("\n")); err != nil { + log.Fatalf("Writing failed: %v", err) + } + } + offset = b.Stop + + // Insert the include file + if err := insertIncludes(&output, b); err != nil { + log.Fatal(err) + } + } + // Copy the remaining of the original file... + if _, err := output.Write(readme[offset:]); err != nil { + log.Fatalf("Writing remaining content failed: %v", err) + } + + // Write output with same permission as input + file, err := os.OpenFile(inputFilename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm) + if err != nil { + log.Fatalf("Opening output file failed: %v", err) + } + defer file.Close() + if _, err := output.WriteTo(file); err != nil { + log.Fatalf("Writing output file failed: %v", err) + } +} diff --git a/conduit/plugins/exporters/filewriter/README.md b/conduit/plugins/exporters/filewriter/README.md new file mode 100644 index 00000000..6efa7f52 --- /dev/null +++ b/conduit/plugins/exporters/filewriter/README.md @@ -0,0 +1,22 @@ +# File Export Plugin + +Write block data to files. This plugin works with the file rerader plugin to create a simple file-based pipeine. + +## Configuration +```yml @sample.yaml + name: "file_writer" + config: + # BlocksDir is the path to a directory where block data should be stored. + # The directory is created if it doesn't exist. If no directory is provided + # blocks are written to the Conduit data directory. + #block-dir: "/path/to/block/files" + + # FilenamePattern is the format used to write block files. It uses go + # string formatting and should accept one number for the round. + # If the file has a '.gz' extension, blocks will be gzipped. + # Default: "%[1]d_block.json" + filename-pattern: "%[1]d_block.json" + + # DropCertificate is used to remove the vote certificate from the block data before writing files. + drop-certificate: true +``` diff --git a/conduit/plugins/exporters/filewriter/file_exporter_config.go b/conduit/plugins/exporters/filewriter/file_exporter_config.go index 17ff2c88..43b643c6 100644 --- a/conduit/plugins/exporters/filewriter/file_exporter_config.go +++ b/conduit/plugins/exporters/filewriter/file_exporter_config.go @@ -1,6 +1,7 @@ package filewriter //go:generate go run ../../../../cmd/conduit-docs/main.go ../../../../conduit-docs/ +//go:generate go run ../../../../cmd/readme_config_includer/generator.go //PluginName: conduit_exporters_filewriter diff --git a/conduit/plugins/exporters/filewriter/sample.yaml b/conduit/plugins/exporters/filewriter/sample.yaml index 89774c4f..c7513c36 100644 --- a/conduit/plugins/exporters/filewriter/sample.yaml +++ b/conduit/plugins/exporters/filewriter/sample.yaml @@ -13,4 +13,3 @@ # DropCertificate is used to remove the vote certificate from the block data before writing files. drop-certificate: true - diff --git a/conduit/plugins/exporters/postgresql/README.md b/conduit/plugins/exporters/postgresql/README.md new file mode 100644 index 00000000..84796a1f --- /dev/null +++ b/conduit/plugins/exporters/postgresql/README.md @@ -0,0 +1,44 @@ +# PostgreSQL Exporter + +Write block data to a postgres database. + +The database maintained by this plugin is designed to serve the Indexer API. + +## Connection string + +The connection string is defined by the [pgx](https://github.com/jackc/pgconn) database driver. + +For most deployments, you can use the following format: +``` +host={url} port={port} user={user} password={password} dbname={db_name} sslmode={enable|disable} +``` + +For additional details, refer to the [parsing documentation here](https://pkg.go.dev/github.com/jackc/pgx/v4/pgxpool@v4.11.0#ParseConfig). + +## Data Pruning + +The delete-task prunes old transactions according to its configuration. This can be used to limit the size of the database. + +## Configuration +```yml @sample.yaml + name: postgresql + config: + # Pgsql connection string + # See https://github.com/jackc/pgconn for more details + connection-string: "host= port=5432 user= password= dbname=" + + # Maximum connection number for connection pool + # This means the total number of active queries that can be running + # concurrently can never be more than this + max-conn: 20 + + # The delete task prunes old transactions according to its configuration. + # By default transactions are not deleted. + delete-task: + # Interval used to prune the data. The values can be -1 to run at startup, + # 0 to disable, or N to run every N rounds. + interval: 0 + + # Rounds to keep + rounds: 100000 +``` \ No newline at end of file diff --git a/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go b/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go index 2f632da4..4ccb4530 100644 --- a/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go +++ b/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go @@ -1,6 +1,7 @@ package postgresql //go:generate go run ../../../../cmd/conduit-docs/main.go ../../../../conduit-docs/ +//go:generate go run ../../../../cmd/readme_config_includer/generator.go import ( "github.com/algorand/conduit/conduit/plugins/exporters/postgresql/util" diff --git a/conduit/plugins/importers/algod/README.md b/conduit/plugins/importers/algod/README.md new file mode 100644 index 00000000..cff60c27 --- /dev/null +++ b/conduit/plugins/importers/algod/README.md @@ -0,0 +1,45 @@ +# Algod Import Plugin + +This plugin imports block data from an algod node. Fetch blocks data from the [algod REST API](https://developer.algorand.org/docs/rest-apis/algod/v2/). + +## Features + +### Automatic Fast Catchup + +If an admin API token and catchpoint are set, the plugin will automatically run fast catchup on startup if the node is behind the current pipeline round. + +### Follower Node Orchestration + +When configured to work with a follower node, this plugin fully automates the management of the follower node. The sync round will be configured according to the pipeline round, and will be advanced as data is importer. + +When using a follower node, ledger state delta objects are provided to the processors and exporter. This data contains detailed state transition information which is necessary for some processors and exporters. + +## Configuration +```yml @sample.yaml + name: algod + config: + # The mode of operation, either "archival" or "follower". + # * archival mode allows you to start processing on any round but does not + # contain the ledger state delta objects required for the postgres writer. + # * follower mode allows you to use a lightweight non-archival node as the + # data source. In addition, it will provide ledger state delta objects to + # the processors and exporter. + mode: "follower" + + # Algod API address. + netaddr: "http://url:port" + + # Algod API token. + token: "" + + # Algod catchpoint catchup arguments + catchup-config: + # The catchpoint to use when running fast catchup. Select an appropriate catchpoint for your deployment. + # They are published in the following locations: + # mainnet: https://algorand-catchpoints.s3.us-east-2.amazonaws.com/consolidated/mainnet_catchpoints.txt + # betanet: https://algorand-catchpoints.s3.us-east-2.amazonaws.com/consolidated/betanet_catchpoints.txt + # testnet: https://algorand-catchpoints.s3.us-east-2.amazonaws.com/consolidated/testnet_catchpoints.txt + catchpoint: "" + # Algod Admin API Token + admin-token: "" +``` \ No newline at end of file diff --git a/conduit/plugins/importers/algod/algod_importer_config.go b/conduit/plugins/importers/algod/algod_importer_config.go index e35bfb41..1a354ef2 100644 --- a/conduit/plugins/importers/algod/algod_importer_config.go +++ b/conduit/plugins/importers/algod/algod_importer_config.go @@ -1,6 +1,7 @@ package algodimporter //go:generate go run ../../../../cmd/conduit-docs/main.go ../../../../conduit-docs/ +//go:generate go run ../../../../cmd/readme_config_includer/generator.go //Name: conduit_importers_algod diff --git a/conduit/plugins/importers/algod/sample.yaml b/conduit/plugins/importers/algod/sample.yaml index e0f76c0b..9637f6e0 100644 --- a/conduit/plugins/importers/algod/sample.yaml +++ b/conduit/plugins/importers/algod/sample.yaml @@ -2,14 +2,14 @@ config: # The mode of operation, either "archival" or "follower". # * archival mode allows you to start processing on any round but does not - # contain the ledger state delta objects required for the postgres writer. + # contain the ledger state delta objects required for the postgres writer. # * follower mode allows you to use a lightweight non-archival node as the - # data source. In addition, it will provide ledger state delta objects to - # the processors and exporter. + # data source. In addition, it will provide ledger state delta objects to + # the processors and exporter. mode: "follower" # Algod API address. - netaddr: "http://url" + netaddr: "http://url:port" # Algod API token. token: "" diff --git a/conduit/plugins/importers/filereader/README.md b/conduit/plugins/importers/filereader/README.md new file mode 100644 index 00000000..1595d12f --- /dev/null +++ b/conduit/plugins/importers/filereader/README.md @@ -0,0 +1,26 @@ +# File Import Plugin + +Read files from a directory and import them as blocks. This plugin works with the file exporter plugin to create a simple file-based pipeline. + +## Configuration +```yml @sample.yaml + name: file_reader + config: + # BlocksDir is the path to a directory where block data should be stored. + # The directory is created if it doesn't exist. If no directory is provided + # blocks are written to the Conduit data directory. + #block-dir: "/path/to/directory" + + # RetryDuration controls the delay between checks when the importer has + # caught up and is waiting for new blocks to appear. + retry-duration: "5s" + + # RetryCount controls the number of times to check for a missing block + # before generating an error. The retry count and retry duration should + # be configured according the expected round time. + retry-count: 5 + + # FilenamePattern is the format used to find block files. It uses go string + # formatting and should accept one number for the round. + filename-pattern: "%[1]d_block.json" +``` diff --git a/conduit/plugins/importers/filereader/filereader_config.go b/conduit/plugins/importers/filereader/filereader_config.go index 69407fd8..bd4f7352 100644 --- a/conduit/plugins/importers/filereader/filereader_config.go +++ b/conduit/plugins/importers/filereader/filereader_config.go @@ -1,6 +1,7 @@ package fileimporter //go:generate go run ../../../../cmd/conduit-docs/main.go ../../../../conduit-docs/ +//go:generate go run ../../../../cmd/readme_config_includer/generator.go import "time" diff --git a/docs/tutorials/FilterDeepDive.md b/conduit/plugins/processors/filterprocessor/README.md similarity index 79% rename from docs/tutorials/FilterDeepDive.md rename to conduit/plugins/processors/filterprocessor/README.md index d85e4fa4..69dc27ce 100644 --- a/docs/tutorials/FilterDeepDive.md +++ b/conduit/plugins/processors/filterprocessor/README.md @@ -1,11 +1,9 @@ -## Filtering Transactions in Conduit +# Transaction Filter Processor -### Intro -Conduit provides individual documentation for each plugin in [docs/conduit/plugins](./plugins). However, the filter -processor in particular has a complex set of features which empower users to search for data within Transactions. -This document will go through some of those features in detail, their use cases, and show some examples. +This plugin allows you to filter transactions based on their contents. This is useful for filtering out transactions +to the specific ones required for your application. -### Filters +## Filters Filters have two top level logical operators, `any` and `all`. These are used to match "sub-expressions" specified in the filters. For any set of expressions, e1, e2, e3, ... `any(e1,e2,e3,...eN)` will return @@ -17,11 +15,11 @@ sub-expression matches. If there are multiple top level filters defined in the configuration, the transaction match is combined with an AND operation. Meaning the transaction must be matched by all defined filters in order to pass through the filter. -### Sub-Expressions +## Sub-Expressions So, what defines a sub-expression? -The sub-expression consists of 3 components. -#### `tag` +The sub-expression consists of 3 components. +### `tag` The tag identifies the field to attempt to match. The fields derive their tags according to the [official reference docs](https://developer.algorand.org/docs/get-details/transactions/transactions/). You can also attempt to match against the `ApplyData`. A complete list of supported tags can be found [here](Filter_tags.md). @@ -39,9 +37,9 @@ Example: - tag: 'txn.amt' # Matches the amount of a payment transaction ``` -#### `expression-type` +### `expression-type` The expression type is a selection of one of the available methods for evaluating the expression. The current list of -types is +types is * `exact`: exact match for string values. * `regex`: applies regex rules to the matching. * `less-than` applies numerical less than expression. @@ -55,12 +53,32 @@ You must use the proper expression type for the field your tag identifies based For example, do not use a numerical expression type on a string field such as address. -#### `expression` +### `expression` The expression is the data against which each field will be compared. This must be compatible with the data type of the expected field. For string fields you can also use the `regex` expression type to interpret the input of the -expression as a regex. +expression as a regex. + + +## Configuration +```yml @sample.yaml +name: filter_processor +config: + # Whether or not the expression searches inner transactions for matches. + search-inner: true + + # Whether or not to include the entire transaction group when the filter + # conditions are met. + omit-group-transactions: true + + # The list of filter expressions to use when matching transactions. + filters: + - any: + - tag: txn.rcv + expression-type: exact + expression: "ADDRESS" +``` -### Examples +## Examples Find transactions w/ fee greater than 1000 microalgos ```yaml @@ -92,7 +110,7 @@ filters: expression: "MYAPPID" ``` -Find transactions, including inner transactions, sent by "FOO". +Find transactions, including inner transactions, sent by "FOO". ```yaml search-inner: true filters: diff --git a/conduit/plugins/processors/filterprocessor/config.go b/conduit/plugins/processors/filterprocessor/config.go index 54b92c9b..c7c382ea 100644 --- a/conduit/plugins/processors/filterprocessor/config.go +++ b/conduit/plugins/processors/filterprocessor/config.go @@ -2,6 +2,7 @@ package filterprocessor //go:generate go run ../../../../cmd/conduit-docs/main.go ../../../../conduit-docs/ +//go:generate go run ../../../../cmd/readme_config_includer/generator.go import ( "github.com/algorand/conduit/conduit/plugins/processors/filterprocessor/expression" diff --git a/docs/Configuration.md b/docs/Configuration.md index fea7abc0..9ede361b 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -50,5 +50,5 @@ exporter: ## Plugin configuration -See [plugin list](plugins/home.md) for details. Each plugin is identified by a `name`, and provided the `config` during initialization. +See plugins documentation for details. diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index 07309d1f..e95ccf54 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -36,9 +36,5 @@ plugins and how to include these plugins in your pipeline's configuration file s ## Set up Conduit for the Indexer API [How to configure algod, PostgreSQL and Conduit as an Indexer API backend.](./tutorials/IndexerWriter.md) -## Using the Transaction Filter Plugin - -[A deep dive into the transaction filter plugin.](./tutorials/FilterDeepDive.md) - ## Writing Block Data to the Filesystem [Use the file exporter to write data to files.](./tutorials/WritingBlocksToFile.md) diff --git a/docs/plugins/algod.md b/docs/plugins/algod.md deleted file mode 100644 index a201f057..00000000 --- a/docs/plugins/algod.md +++ /dev/null @@ -1,20 +0,0 @@ -# Algod Importer - -Fetch blocks one by one from the [algod REST API](https://developer.algorand.org/docs/rest-apis/algod/v2/). The node must be configured as an archival node in order to -provide old blocks. - -Block data from the Algod REST API contains the block header, transactions, and a vote certificate. - -# Config -```yaml -importer: - name: algod - config: - - netaddr: "algod URL" - token: "algod REST API token" - # This is an optional section which will enable Conduit to run fast catchup on your behalf using the provided catchpoint - catchup-config: - catchpoint: "the catchpoint to use for fast catchup" - admin-token: "the algod admin API token (used to run catchpoint catchup)" -``` - diff --git a/docs/plugins/file_writer.md b/docs/plugins/file_writer.md deleted file mode 100644 index 3c93d874..00000000 --- a/docs/plugins/file_writer.md +++ /dev/null @@ -1,20 +0,0 @@ -# Filewriter Exporter - -Write the block data to a file. - -Data is written to one file per block in JSON format. - -By default data is written to the filewriter plugin directory inside the indexer data directory. - -# Config -```yaml -exporter: - - name: file_writer - config: - - block-dir: "override default block data location." - # override the filename pattern. - filename-pattern: "%[1]d_block.json" - # exclude the vote certificate from the file. - drop-certificate: false -``` - diff --git a/docs/plugins/filter_processor.md b/docs/plugins/filter_processor.md deleted file mode 100644 index 6a81ed2a..00000000 --- a/docs/plugins/filter_processor.md +++ /dev/null @@ -1,67 +0,0 @@ -# Filter Processor - -This is used to filter transactions to include only the ones that you want. This may be useful for some deployments -which only require specific applications or accounts. Using the default configuration, results for a matched transaction -would also include its transaction group and search is disable for inner transactions. - -## any / all -One or more top-level operations should be provided. -* any: transactions are included if they match `any` of the nested sub expressions. -* all: transactions are included if they match `all` of the nested sub expressions. - -If `any` and `all` are both provided, the transaction must pass both checks. - -## Sub expressions - -Parts of an expression: -* `tag`: the transaction field being considering. -* `expression-type`: The type of expression. -* `expression`: Input to the expression - -### tag -The full path to a given field. Uses the messagepack encoded names of a canonical transaction. For example: -* `txn.snd` is the sender. -* `txn.amt` is the amount. - -For information about the structure of transactions, refer to the [Transaction Structure](https://developer.algorand.org/docs/get-details/transactions/) documentation. For detail about individual fields, refer to the [Transaction Reference](https://developer.algorand.org/docs/get-details/transactions/transactions/) documentation. - -**Note**: The "Apply Data" information is also available for filtering. These fields are not well documented. Advanced users can inspect raw transactions returned by algod to see what fields are available. - -### expression-type - -What type of expression to use for filtering the tag. -* `exact`: exact match for string values. -* `regex`: applies regex rules to the matching. -* `less-than` applies numerical less than expression. -* `less-than-equal` applies numerical less than or equal expression. -* `greater-than` applies numerical greater than expression. -* `greater-than-equal` applies numerical greater than or equal expression. -* `equal` applies numerical equal expression. -* `not-equal` applies numerical not equal expression. - -### expression - -The input to the expression. A number or string depending on the expression type. - -# Config -```yaml -processors: - - name: filter_processor - config: - filters: - - any - - tag: - expression-type: - expression: - - tag: - expression-type: - expression: - - all - - tag: - expression-type: - expression: - - tag: - expression-type: - expression: -``` - diff --git a/docs/plugins/noop_exporter.md b/docs/plugins/noop_exporter.md deleted file mode 100644 index f9b074c2..00000000 --- a/docs/plugins/noop_exporter.md +++ /dev/null @@ -1,11 +0,0 @@ -# Noop Exporter - -For testing purposes, the noop processor discards any data it receives. - -# Config -```yaml -processors: - - name: noop - config: -``` - diff --git a/docs/plugins/noop_processor.md b/docs/plugins/noop_processor.md deleted file mode 100644 index ed696d0a..00000000 --- a/docs/plugins/noop_processor.md +++ /dev/null @@ -1,11 +0,0 @@ -# Noop Processor - -For testing purposes, the noop processor simply passes the input to the output. - -# Config -```yaml -processors: - - name: noop - config: -``` - diff --git a/docs/plugins/postgresql.md b/docs/plugins/postgresql.md deleted file mode 100644 index 101515e1..00000000 --- a/docs/plugins/postgresql.md +++ /dev/null @@ -1,23 +0,0 @@ -# PostgreSQL Exporter - -Write block data to a postgres database with the Indexer REST API schema. - -## Connection string - -We are using the [pgx](https://github.com/jackc/pgconn) database driver, which dictates the connection string format. - -For most deployments, you can use the following format: -`host={url} port={port} user={user} password={password} dbname={db_name} sslmode={enable|disable}` - -For additional details, refer to the [parsing documentation here](https://pkg.go.dev/github.com/jackc/pgx/v4/pgxpool@v4.11.0#ParseConfig). - -# Config -```yaml -exporter: - - name: postgresql - config: - - connection-string: "postgres connection string" - max-conn: "connection pool setting, maximum active queries" - test: "a boolean, when true a mock database is used" -``` - diff --git a/docs/tutorials/IndexerMigration.md b/docs/tutorials/IndexerMigration.md index 293de911..45e87bae 100644 --- a/docs/tutorials/IndexerMigration.md +++ b/docs/tutorials/IndexerMigration.md @@ -235,7 +235,7 @@ Because our Conduit pipeline will use the Follower node's state delta API, we no volume. It can be removed. ### Step 3: Refactor our Indexer Writer to Conduit -You're free to capture any data you like using Conduit. I'd encourage you to take a look at the [filter processor](./FilterDeepDive.md) and see +You're free to capture any data you like using Conduit. I'd encourage you to take a look at the `filter processor` and see if you can reduce the amount of data you store in your database by removing non-relevant data. If you would like to maintain parity with the legacy Indexer, which stored all data, you can use the `conduit init` command to create diff --git a/docs/tutorials/WritingBlocksToFile.md b/docs/tutorials/WritingBlocksToFile.md index 19a7390d..c2a17411 100644 --- a/docs/tutorials/WritingBlocksToFile.md +++ b/docs/tutorials/WritingBlocksToFile.md @@ -139,7 +139,7 @@ For our use case we'll grab the address of a wallet I've created on testnet, `NV That should give us exactly what we want, a filter that only allows transaction through for which the receiver is my account. However, there is a lot more you can do with the filter processor. To learn more about the possible uses, take -a look at the individual plugin documentation [here](../plugins/filter_processor.md). +a look at the filter plugin documentation. ## Setting up our Exporter @@ -211,4 +211,4 @@ Once we've processed round 26141781, we should see our transaction show up! There are many other existing plugins and use cases for Conduit! Take a look through the documentation and don't hesitate to open an issue if you have a question. If you want to get a deep dive into the different types of filters -you can construct using the filter processor, take a look at our [filter guide](./FilterDeepDive.md). +you can construct using the filter processor, take a look at our filter plugin. diff --git a/go.mod b/go.mod index 9a75a751..ebfa6eb1 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ require ( github.com/stretchr/objx v0.5.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect + github.com/yuin/goldmark v1.5.4 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.17.0 // indirect diff --git a/go.sum b/go.sum index 17079d9d..b88dae19 100644 --- a/go.sum +++ b/go.sum @@ -800,6 +800,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= +github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=