From ca332a0bcfe2bc32ae8b3757101248fe5bd9ea5d Mon Sep 17 00:00:00 2001 From: Tim McMackin Date: Wed, 27 Nov 2024 12:19:03 -0500 Subject: [PATCH 1/6] Script to concatenate MD files (#471) * Script to concatenate MD files * Remove front matter and replace with title H1 * Filter out tutorials * Filter out glossary * Generate file and store on S3 * Change file name to allPageSourceFiles.md * Faster performance by not processing every line * Convert to ES6 * Process to HTML with Remark * Strip markdown instead of processing to HTML * This is a text file now * Fix underscore escaping * Send tutorials to separate file --- .github/workflows/deploy-production.yml | 2 + .gitignore | 2 + package-lock.json | 31 ++++++ package.json | 3 + src/scripts/concatenate.mjs | 134 ++++++++++++++++++++++++ 5 files changed, 172 insertions(+) create mode 100644 src/scripts/concatenate.mjs diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 447a06942..a81342c7a 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -19,6 +19,8 @@ jobs: node-version: 18 - run: npm ci - run: npm run build + - run: npm run concat + - run: mv allPageSourceFiles.txt allTutorials.txt ./build/ - uses: aws-actions/configure-aws-credentials@v2 with: aws-access-key-id: ${{ secrets.AWS_S3_PRODUCTION_KEY_ID }} diff --git a/.gitignore b/.gitignore index d94b8b952..d08f32ec9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ node_modules out build _glossaryBuild +allPageSourceFiles.txt +allTutorials.txt diff --git a/package-lock.json b/package-lock.json index 19815c9b6..fd7750e0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,8 +34,10 @@ "minimist": "1.2.8", "mocha": "10.2.0", "rehype-stringify": "10.0.0", + "remark": "^15.0.1", "remark-parse": "11.0.0", "remark-rehype": "11.0.0", + "strip-markdown": "^6.0.0", "unified": "11.0.4", "unist-util-visit": "5.0.0" }, @@ -14716,6 +14718,22 @@ "node": ">= 0.10" } }, + "node_modules/remark": { + "version": "15.0.1", + "resolved": "https://registry.npmjs.org/remark/-/remark-15.0.1.tgz", + "integrity": "sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A==", + "dev": true, + "dependencies": { + "@types/mdast": "^4.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/remark-directive": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/remark-directive/-/remark-directive-3.0.0.tgz", @@ -15905,6 +15923,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strip-markdown": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-markdown/-/strip-markdown-6.0.0.tgz", + "integrity": "sha512-mSa8FtUoX3ExJYDkjPUTC14xaBAn4Ik5GPQD45G5E2egAmeV3kHgVSTfIoSDggbF6Pk9stahVgqsLCNExv6jHw==", + "dev": true, + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/style-to-object": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-0.4.4.tgz", diff --git a/package.json b/package.json index 1216f2b19..86591c507 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "docusaurus": "docusaurus", "start": "docusaurus start", "build": "node ./src/scripts/process_downloaded_glossary.js && docusaurus build", + "concat": "node ./src/scripts/concatenate.mjs", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", @@ -41,8 +42,10 @@ "minimist": "1.2.8", "mocha": "10.2.0", "rehype-stringify": "10.0.0", + "remark": "^15.0.1", "remark-parse": "11.0.0", "remark-rehype": "11.0.0", + "strip-markdown": "^6.0.0", "unified": "11.0.4", "unist-util-visit": "5.0.0" }, diff --git a/src/scripts/concatenate.mjs b/src/scripts/concatenate.mjs new file mode 100644 index 000000000..a5813da22 --- /dev/null +++ b/src/scripts/concatenate.mjs @@ -0,0 +1,134 @@ +import path from 'path'; +import fs from 'fs'; + +import {remark} from 'remark' +import strip from 'strip-markdown'; + +import { dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +import sidebars from '../../sidebars.js'; +// Map sidebars to output file names +const fileNames = { + documentationSidebar: 'allPageSourceFiles.txt', + tutorialsSidebar: 'allTutorials.txt', +} +const pathsToFilterOut = [ + 'overview/glossary', +]; + +// Given a docusaurus sidebar object, return a list of the local doc IDs in it +function getIdsRecursive(sidebarObject) { + if (typeof sidebarObject === 'string') { + return sidebarObject; + } + if (sidebarObject.constructor.name == "Array") { + return sidebarObject.reduce((list, oneSidebarObj) => + list.concat(getIdsRecursive(oneSidebarObj)), + []); + } + switch(sidebarObject.type) { + case 'category': + return [sidebarObject?.link?.id].concat(getIdsRecursive(sidebarObject.items)); + case 'doc': + return sidebarObject.id; + case 'link': + if (sidebarObject.href && sidebarObject.href.startsWith('http')) { + return null; + } else { + return sidebarObject.href; + } + default: + return null; + } +} + +// Given a doc file ID from the sidebar, get the filename path +async function getFilePath(fileId) { + const mdPath = path.resolve(__dirname, '../../docs', fileId) + '.md'; + try { + await fs.promises.access(mdPath, fs.constants.F_OK); + return mdPath; + } catch { + // Do nothing + } + const mdxPath = mdPath + 'x'; + try { + await fs.promises.access(mdxPath, fs.constants.F_OK); + return mdxPath; + } catch { + console.error("Could not file file with sidebar ID", fileId); + } +} + +// Remove the front matter from an MD file and replace with an H1 +// Got to remove FM because multiple FM blocks break some markdown tools +// Could do this with gray-matter but I don't want to add the dependency +function removeFrontMatter(mdText) { + const lines = mdText.split('\n'); + let inFrontMatter = false; + let doneWithFrontMatter = false; + const h1Regex = /^title:\s+(.*)$/; + let titleLine = ''; + let line = ''; + + while (lines.length > 0) { + line = lines.shift(); + if (line == '---') { + doneWithFrontMatter = inFrontMatter; + inFrontMatter = true; + } + if (inFrontMatter && !doneWithFrontMatter && h1Regex.test(line)) { + const result = h1Regex.exec(line); + titleLine = '# ' + result[1]; + } + if (line != '---' && doneWithFrontMatter) { + return [titleLine, ''].concat(lines).join('\n'); + } + } +} + +async function concatSidebar(sidebarName) { + const outputPath = path.resolve(__dirname, '../../', fileNames[sidebarName]); + + // Remove old concatenated file if it exists + try { + await fs.promises.access(outputPath, fs.constants.F_OK); + await fs.promises.unlink(outputPath); + } catch { + // Do nothing because the file does not exist + } + + const allMdIds = getIdsRecursive(sidebars[sidebarName]) + .filter((id) => !pathsToFilterOut.includes(id)) + .filter((item) => item); + + // Find the matching file paths + const allFilePaths = await Promise.all(allMdIds.map(getFilePath)); + + // Read and concat the files in TOC order + await allFilePaths.reduce(async (previousPromise, oneFilePath) => { + await previousPromise; + const markdownText = removeFrontMatter(await fs.promises.readFile(oneFilePath, 'utf8')); + const oneFileText = await remark() + .use(strip) + .process(markdownText); + // Fix strip plugin escaping `_` as `\_` + const oneFileTextFixEscaped = String(oneFileText).replaceAll('\\\_', '_'); + return fs.promises.appendFile(outputPath, oneFileTextFixEscaped + '\n\n'); + }, Promise.resolve()); + + console.log(`Wrote concatenated file for sidebar ${sidebarName} to ${outputPath}`); +} + +// Concatenate the sidebars listed in fileNames to separate single files +async function concatSidebars() { + const sidebarNames = Object.keys(sidebars); + await Promise.all(sidebarNames + .filter((oneSidebarName) => Object.keys(fileNames).includes(oneSidebarName)) + .map(concatSidebar)); +} + +concatSidebars(); From ac9f34576399fb0290618e75a00dd6c604765955 Mon Sep 17 00:00:00 2001 From: Tim McMackin Date: Thu, 28 Nov 2024 07:51:26 -0500 Subject: [PATCH 2/6] Clarify DAL node networking, just covering the basics (#475) --- docs/tutorials/join-dal-baker/run-dal-node.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/join-dal-baker/run-dal-node.md b/docs/tutorials/join-dal-baker/run-dal-node.md index 6abb5e81c..fd2896f78 100644 --- a/docs/tutorials/join-dal-baker/run-dal-node.md +++ b/docs/tutorials/join-dal-baker/run-dal-node.md @@ -2,7 +2,7 @@ title: "Step 3: Run an Octez DAL node" authors: Tezos core developers, Tim McMackin last_update: - date: 18 September 2024 + date: 27 November 2024 --- The DAL node is responsible for temporarily storing data and providing it to bakers and Smart Rollups. @@ -26,8 +26,8 @@ For example, this command initializes the DAL node with the address of a local ` This, too, may take some time to launch the first time because it needs to generate a new identity file, this time for the DAL network. - To set the address and port that the node listens on, pass the `--net-addr` argument. - By default, it listens on port 11732 on all available network interfaces, equivalent to `--net-addr 0.0.0.0:11732`. + If you need to change the address or port that the DAL node listens for connections to other nodes on, pass the `--public-addr` argument. + By default, it listens on port 11732 on all available network interfaces, equivalent to `--public-addr 0.0.0.0:11732`. 1. Verify that the DAL node is connected to the DAL network by running this command: From 727e1a2a689841a85299540c4817ccf046ed0475 Mon Sep 17 00:00:00 2001 From: Tim McMackin Date: Thu, 28 Nov 2024 09:53:37 -0500 Subject: [PATCH 3/6] Clean up escapes (#474) --- src/scripts/concatenate.mjs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/scripts/concatenate.mjs b/src/scripts/concatenate.mjs index a5813da22..f62df6e19 100644 --- a/src/scripts/concatenate.mjs +++ b/src/scripts/concatenate.mjs @@ -90,6 +90,20 @@ function removeFrontMatter(mdText) { } } +// Unified tools put escapes in places +// We could get around this by forking mdast-util-to-string as described here: +// https://github.com/remarkjs/strip-markdown/issues/28#issuecomment-1290847745 +function cleanUpEscapes(text) { + return text + // Fix stripped markdown escaping such as `_` to `\_` + .replaceAll('\\\_', '_') + .replaceAll('\\<', '<') + .replaceAll('\\[', '[') + .replaceAll(/^\\-/gm, '-') + .replaceAll('\\.', '.') + .replaceAll('\\*', '*'); +} + async function concatSidebar(sidebarName) { const outputPath = path.resolve(__dirname, '../../', fileNames[sidebarName]); @@ -115,8 +129,7 @@ async function concatSidebar(sidebarName) { const oneFileText = await remark() .use(strip) .process(markdownText); - // Fix strip plugin escaping `_` as `\_` - const oneFileTextFixEscaped = String(oneFileText).replaceAll('\\\_', '_'); + const oneFileTextFixEscaped = cleanUpEscapes(String(oneFileText)); return fs.promises.appendFile(outputPath, oneFileTextFixEscaped + '\n\n'); }, Promise.resolve()); From ec567ba41e0adda65cae33e9f28711255f56291a Mon Sep 17 00:00:00 2001 From: Tim McMackin Date: Mon, 2 Dec 2024 08:55:02 -0500 Subject: [PATCH 4/6] Review "What makes Tezos different" page (#481) * Add link to governance topic * Remove info about formal verification b/c no one ever talks about it * Update with info about user delegating and staking * Mention layer 2 * Enshrined is a good word to use here, not sure about the language --- docs/overview/tezos-different.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/overview/tezos-different.md b/docs/overview/tezos-different.md index 319391606..d796f3090 100644 --- a/docs/overview/tezos-different.md +++ b/docs/overview/tezos-different.md @@ -1,26 +1,25 @@ --- title: What makes Tezos different? last_update: - date: 13 September 2023 + date: 29 November 2024 --- Here are some of the features that make Tezos different from other blockchains: ## Tezos can upgrade itself -Tezos has a built-in capability to upgrade itself, which allows the network to evolve without requiring a hard fork. Anyone can propose an upgrade to the protocol and have it adopted by the network without compromising the platform's stability or causing fragmentation. This feature allows Tezos to adapt regularly to new technologies and to address user needs rapidly. +Tezos has a built-in capability to upgrade itself, which allows the network to evolve without requiring a hard fork. Anyone can propose an upgrade to the protocol and have it adopted by the network without compromising the platform's stability or causing fragmentation. This feature allows Tezos to adapt regularly to new technologies and to address user needs rapidly. For more information, see [Governance](/architecture/governance). ## Everyone can participate in governance Anyone who holds XTZ — the chain's native token — can propose changes to how Tezos works, such as changes to gas fees and block times, new features such as Smart Rollups, or even major changes like how the consensus mechanism works. -## Formal verification ensures trust and code quality - -*Formal verification* is a process that ensures that a smart contract does what it says it does and has no side effects. Formal verification reduces errors, bugs, and security vulnerabilities in contracts and allows users to trust them. For more information, see [Formal Verification](https://opentezos.com/formal-verification) on opentezos.com. - ## Tezos uses proof of stake -The proof-of-stake consensus model eliminates the need for high energy use, making it the "green" choice for blockchains. Instead of competing to achieve consensus as in proof-of-work models, Tezos nodes (called *bakers*) stake Tezos tokens to earn the right to create blocks and receive rewards. Users who want to participate without running a node themselves can delegate tokens to a baker for a share of the rewards. The bakers keep control of their tokens in that they can unstake them later; and delegators keep complete control of their tokens. Tezos's approach to consensus has been described as [Liquid Proof of Stake](https://medium.com/tezos/liquid-proof-of-stake-aec2f7ef1da7). +The proof-of-stake consensus model eliminates the need for high energy use, making it the "green" choice for blockchains. Instead of competing to achieve consensus as in proof-of-work models, Tezos nodes (called *bakers*) stake Tezos tokens to earn the right to create blocks and receive rewards. Tezos's approach to consensus has been described as [Liquid Proof of Stake](https://medium.com/tezos/liquid-proof-of-stake-aec2f7ef1da7). + +Users who want to participate without running a node themselves can delegate and stake tokens to a baker for a share of the baker's rewards. Users retain full control of their delegated and staked tokens; delegated tokens remain liquid and staked tokens are frozen for a short time but remain in the user's account and can be made liquid after a short delay. + The proof-of-stake model improves scalability and encourages cooperation via incentives. It also increases the cost of 51% attacks and avoids environmentally wasteful proof-of-work. Tezos launched in June 2018 as one of the first major proof-of-stake networks. @@ -29,3 +28,9 @@ For more information about how Tezos handles proof of stake, see https://tezos.g ## Tezos accepts multiple languages Tezos provides a few different languages for developers to choose from, according to their use case, including versions of Python and JavaScript/TypeScript. For more information, see [Languages](/smart-contracts/languages/). + +## Tezos has a robust layer 2 ecosystem + +Tezos provides tools that allow high scalability on a layer above the primary Tezos blockchain, known as layer 2. [Smart Rollups](/architecture/smart-rollups) can run large amounts of logic and handle large amounts of data in a separate environment without slowing Tezos itself down. The [Data Availability Layer](/architecture/data-availability-layer) provides high-throughput data to Smart Rollups. + +The framework for these layer 2 systems is enshrined in the layer 1 protocol. All layer 2 activity is secured by verifiable commitments and attestations on layer 1. From 230ecb38c6165b206e0a16f4eac95719f15092eb Mon Sep 17 00:00:00 2001 From: Tim McMackin Date: Mon, 2 Dec 2024 08:55:46 -0500 Subject: [PATCH 5/6] Some small updates to this tutorial (#479) --- docs/tutorials/create-an-nft/nft-tznft.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/tutorials/create-an-nft/nft-tznft.md b/docs/tutorials/create-an-nft/nft-tznft.md index ae369cd35..98dbda075 100644 --- a/docs/tutorials/create-an-nft/nft-tznft.md +++ b/docs/tutorials/create-an-nft/nft-tznft.md @@ -2,7 +2,7 @@ title: Create NFTs from the command line authors: 'Sol Lederer, Tim McMackin' last_update: - date: 18 September 2023 + date: 28 November 2024 --- This tutorial covers how to create a collection of NFTs on Tezos and manipulate them using the `tznft` command-line tool. @@ -27,7 +27,7 @@ NFTs can represent ownership over digital or physical assets like virtual collec Like other types of Tezos tokens, a collection of NFTs is managed by a smart contract. The smart contract defines what information is in each token and how the tokens behave, such as what happens when a user transfers an NFT to another user. -In this tutorial, you create NFTs that comply with the FA2 standard (formally known as the [TZIP-12](https://gitlab.com/tezos/tzip/-/blob/master/proposals/tzip-12/tzip-12.md) standard), the current standard for tokens on Tezos. +In this tutorial, you create NFTs that comply with the [FA2](/architecture/tokens/FA2) standard, a popular and flexible standard for tokens on Tezos. The FA2 standard creates a framework for how tokens behave on Tezos, including fungible, non-fungible, and other types of tokens. It provides a standard API to transfer tokens, check token balances, manage operators (addresses that are permitted to transfer tokens on behalf of the token owner), and manage token metadata. @@ -35,7 +35,7 @@ It provides a standard API to transfer tokens, check token balances, manage oper To run this tutorial you need Node.JS, NPM, and Docker Desktop to install and use the `tznft` CLI tool, which helps you create and test NFT collections on Tezos. -- Install Node.JS version 18 (not 20) and NPM. +- Install Node.JS version 18 (no later) and NPM. See https://nodejs.org/. You can verify that they are installed by running these commands: @@ -119,7 +119,7 @@ The first step in creating NFTs is to create local metadata files that describe ``` The new metadata file is named `my_collection.json` and has information such as the name, description, home page, and creator of the collection. - It also includes the interfaces that the NFTs support, including the TZIP-12 interface that was mentioned earlier. + It also includes the interfaces that the NFTs support, including the TZIP-12 interface that is the basis for the FA2 standard. 1. Optional: Edit the `my_collection.json` file to put your information in the `name`, `description`, and `authors` fields. @@ -476,6 +476,9 @@ Similarly, you can use the same collection alias because `tznft` keeps aliases s tznft create-collection my-account --meta_file my_collection.json --alias my_collection ``` + If you see an error that says that the request failed with the message `getaddrinfo ENOTFOUND`, the problem could be that the Tezos testnet node is not available. + If you see this error, open the `tznft.json` file, go to the `testnet` section, and change the `providerUrl` field to the URL of a testnet RPC node from the site https://teztnets.com. + 1. Mint the tokens on the testnet. The command is the same as for the sandbox: @@ -483,7 +486,8 @@ The command is the same as for the sandbox: tznft mint my-account my_collection --tokens '1, ipfs://abcde12345' ``` - You can add more NFTs until you freeze the collection. + You must use the same account alias that you used to create the collection because that account becomes the administrator of the collection. + Just like in the sandbox, you can add more NFTs until you freeze the collection. 1. View your token balances. The command is the same as for the sandbox: From 0a6e7d54a3fa61ab629192422d1a578e1da92a0c Mon Sep 17 00:00:00 2001 From: Tim McMackin Date: Mon, 2 Dec 2024 13:02:32 -0500 Subject: [PATCH 6/6] Update codeowners for current team working on docs (#482) * Remove codeowners file because this team no longer exists * Use tur-docs instead * techrel-docs --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 613176905..0a18762c3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @trilitech/developer-success +* @trilitech/techrel-docs