From c5df86cc2d573a118ef0b52e852722a184f48c82 Mon Sep 17 00:00:00 2001 From: A-F-V Date: Wed, 5 Jul 2023 22:11:55 +0100 Subject: [PATCH 1/4] Update FrontMatterManager.ts --- src/include/FrontMatterManager.ts | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/src/include/FrontMatterManager.ts b/src/include/FrontMatterManager.ts index a8f6701..2d4cdc9 100644 --- a/src/include/FrontMatterManager.ts +++ b/src/include/FrontMatterManager.ts @@ -7,35 +7,7 @@ export default class FrontMatterManager { constructor(arcana: ArcanaPlugin) { this.arcana = arcana; } - // TODO: give type: (Map=>void) - async processFrontMatter(file: TFile, transformation: any): Promise { - await this.arcana.app.fileManager.processFrontMatter(file, frontMatter => { - // Get the current front matter, defaulting to empty if it doesn't exist - let arcanaFrontMatter = new Map(); - if (frontMatter.arcana !== undefined) { - arcanaFrontMatter = new Map(Object.entries(frontMatter.arcana)); - } - // Apply the transformation - transformation(arcanaFrontMatter); - // Save the result - frontMatter.arcana = Object.fromEntries(arcanaFrontMatter.entries()); - }); - } - /* - async getArcana(file: TFile, key: string): Promise { - let result = null; - await this.processFrontMatter(file, (arcanaData: any) => { - result = arcanaData.get(key); - }); - return result; - } - async setArcana(file: TFile, key: string, value: any): Promise { - await this.processFrontMatter(file, (arcanaData: any) => { - arcanaData.set(key, value); - }); - } - */ async set(file: TFile, key: string, value: any): Promise { await this.arcana.app.fileManager.processFrontMatter(file, frontMatter => { frontMatter[key] = value; @@ -59,8 +31,7 @@ export default class FrontMatterManager { await this.arcana.app.fileManager .processFrontMatter(file, frontmatter => { - console.log(frontmatter); - //tags = parseFrontMatterTags(frontmatter); + tags = parseFrontMatterTags(frontmatter); }) .catch(e => {}); From 7372aad29a799a999a17ae2a82caeda8e21756c1 Mon Sep 17 00:00:00 2001 From: A-F-V Date: Wed, 5 Jul 2023 22:33:28 +0100 Subject: [PATCH 2/4] Settings for Darwin added --- src/plugins/Darwin/Darwin.ts | 136 +++++++++++++++++++++++++++++++---- 1 file changed, 123 insertions(+), 13 deletions(-) diff --git a/src/plugins/Darwin/Darwin.ts b/src/plugins/Darwin/Darwin.ts index a0240f6..0319a8f 100644 --- a/src/plugins/Darwin/Darwin.ts +++ b/src/plugins/Darwin/Darwin.ts @@ -2,6 +2,7 @@ import { Editor, MarkdownView, Notice, + Setting, TAbstractFile, TFile, TFolder, @@ -12,9 +13,31 @@ import { removeFrontMatter } from 'src/utilities/DocumentCleaner'; import ArcanaPluginBase from 'src/components/ArcanaPluginBase'; import FrontMatterManager from 'src/include/FrontMatterManager'; +enum TagStyle { + None, + PascalCase, + CamelCase, + SnakeCase, + KebabCase, + TitleCase, +} + +type DarwinSettings = { + minimum_tag_count_to_present: number; + only_show_existing_tags: boolean; + max_tags_to_show: number; + tag_style: TagStyle; +}; + +const DEFAULT_SETTINGS: DarwinSettings = { + minimum_tag_count_to_present: 1, + only_show_existing_tags: false, + max_tags_to_show: 4, + tag_style: TagStyle.None, +}; export default class DarwinPlugin extends ArcanaPluginBase { private arcana: ArcanaPlugin; - private setting: { folder: string }; + private setting: DarwinSettings = DEFAULT_SETTINGS; private tagCountCache: [string, number][] | null = null; public constructor(arcana: ArcanaPlugin) { @@ -22,9 +45,6 @@ export default class DarwinPlugin extends ArcanaPluginBase { this.arcana = arcana; } - // TODO: #44: Setting for max number of tags to show - private MAX_TAGS_TO_SHOW = 4; - private async getTagsForFile(file: TFile): Promise { const fmm = new FrontMatterManager(this.arcana); // Get the tags from the front matter @@ -83,11 +103,11 @@ export default class DarwinPlugin extends ArcanaPluginBase { // Check if tag is empty if (tag.length === 0) return false; // Check if tag has a non-numerical character - if (!/[a-zA-Z_\-\/]/.test(tag)) return false; + if (!/[a-zA-Z_\-/]/.test(tag)) return false; // Check if tag has a space if (/\s/.test(tag)) return false; // Check if the tag has only valid characters - if (!/^[a-zA-Z0-9_\-\/]+$/.test(tag)) return false; + if (!/^[a-zA-Z0-9_\-/]+$/.test(tag)) return false; return true; } @@ -107,7 +127,7 @@ export default class DarwinPlugin extends ArcanaPluginBase { .filter(tag => this.tagCountCache?.map(t => t[0]).includes(tag)); const uniqueTags = [...new Set(tags)]; if (uniqueTags.length === 0) return []; - if (uniqueTags.length > this.MAX_TAGS_TO_SHOW) return []; + if (uniqueTags.length > this.setting.max_tags_to_show) return []; return uniqueTags; } @@ -118,6 +138,9 @@ export default class DarwinPlugin extends ArcanaPluginBase { } public async onload() { + this.setting = + this.arcana.settings.PluginSettings['Darwin'] ?? DEFAULT_SETTINGS; + // Get lits of all tags periodically await this.getAllTagsInVault(); this.arcana.registerInterval( @@ -168,6 +191,87 @@ export default class DarwinPlugin extends ArcanaPluginBase { ); } + public addSettings(containerEl: HTMLElement) { + containerEl.createEl('h2', { text: 'Darwin' }); + + new Setting(containerEl) + .setName('Max tags to add') + .setDesc('The max tags Darwin should recommend') + .addText(text => { + text + .setPlaceholder('4') + .setValue(this.setting.max_tags_to_show.toString()) + .onChange(async (value: string) => { + const limit = parseInt(value); + if (isNaN(limit)) { + new Notice(`Darwin: Invalid number ${value} as max tags`); + return; + } + this.setting.max_tags_to_show = limit; + this.arcana.settings.PluginSettings['Darwin'] = this.setting; + await this.arcana.saveSettings(); + }); + }); + + new Setting(containerEl) + .setName('Only Existing Tags') + .setDesc( + 'If enabled, Darwin will only suggest tags that already exist in the vault' + ) + .addToggle(toggle => { + toggle + .setValue(this.setting.only_show_existing_tags) + .onChange(async (value: boolean) => { + this.setting.only_show_existing_tags = value; + this.arcana.settings.PluginSettings['Darwin'] = this.setting; + await this.arcana.saveSettings(); + }); + }); + + new Setting(containerEl) + .setName('Min tag count to show to Darwin') + .setDesc( + 'The minimum number of times a tag must appear in the vault to be shown to Darwin. Darwin does better with fewer tags to choose from.' + ) + .addText(text => { + text + .setPlaceholder('1') + .setValue(this.setting.minimum_tag_count_to_present.toString()) + .onChange(async (value: string) => { + const limit = parseInt(value); + if (isNaN(limit)) { + new Notice( + `Darwin: Invalid number ${value} as min tag count to show` + ); + return; + } + this.setting.minimum_tag_count_to_present = limit; + this.arcana.settings.PluginSettings['Darwin'] = this.setting; + await this.arcana.saveSettings(); + }); + }); + + new Setting(containerEl) + .setName('New Tag Style') + .setDesc('The style of new tags that Darwin will suggest') + .addDropdown(dropdown => { + dropdown + .addOption('None', 'None') + .addOption('KebabCase', 'kebab-case') + .addOption('SnakeCase', 'snake_case') + .addOption('CamelCase', 'camelCase') + .addOption('PascalCase', 'PascalCase') + .setValue(this.setting.tag_style.toString()) + .onChange(async (value: string) => { + // Parse the value as a TagStyle + const tagStyle = TagStyle[value as keyof typeof TagStyle]; + this.setting.tag_style = tagStyle; + this.arcana.settings.PluginSettings['Darwin'] = this.setting; + await this.arcana.saveSettings(); + }); + }); + } + public async onunload() {} private async autoTagFile(file: TFile) { @@ -183,8 +287,6 @@ export default class DarwinPlugin extends ArcanaPluginBase { await fmm.setTags(file, tagsInFrontMatter.concat(tagsToAdd)); } - // TODO: #45: Setting to specify the style explictly - private async askDarwin(file: TFile): Promise { const purpose = `You are an AI designed to select additional tags to add to a given file. You select from handful of EXISTING TAGS that you will suggest should be added.`; @@ -198,7 +300,7 @@ export default class DarwinPlugin extends ArcanaPluginBase { const rules = `1. Return a list of additional tags to add that are most relevant to the file. The list should be space seperated. For example: 'tag1 tag2 tag3'. 2. You should only return the list of additional tags and nothing else. Do not give any preamble or other text. - 3. The length of the list of additional tags must be less than or equal to ${this.MAX_TAGS_TO_SHOW}. + 3. The length of the list of additional tags must be less than or equal to ${this.setting.max_tags_to_show}. 4. The list of additional tags must not contain any tags that are already present in the file. 5. Tags must be be valid according to the tag format. 6. Only suggest tags that are in the EXISTING TAGS list.`; @@ -206,8 +308,6 @@ export default class DarwinPlugin extends ArcanaPluginBase { const context = ` [Purpose] ${purpose}. - [EXISTING TAGS] - ${existing_tags} [Tag format] ${tag_format} [Rules] @@ -222,11 +322,21 @@ export default class DarwinPlugin extends ArcanaPluginBase { const cleanedText = removeFrontMatter(documentText); // Get the tags in the file: const tagsInFile = await this.getTagsForFile(file); - let details = `Title of file: ${title}\n\nText in file: ${cleanedText}\n\n`; + + let details = ` + [EXISTING TAGS] + ${existing_tags} + [Title] + ${title} + [Text] + ${cleanedText} + `; if (tagsInFile.length > 0) details += `Tags present in file: ${tagsInFile.join(' ')}`; const response = await this.arcana.complete(details, context); + console.log(`Details: ${details}`); + console.log(`Response: ${response}`); return this.getAdditionalTagsFromResponse(response, tagsInFile); } From 791cff18c843d5f6d2fc26084719e5ea0a7ab86f Mon Sep 17 00:00:00 2001 From: A-F-V Date: Wed, 5 Jul 2023 23:35:30 +0100 Subject: [PATCH 3/4] Added several features to Darwin --- src/include/FrontMatterManager.ts | 4 +- src/plugins/Darwin/Darwin.ts | 218 +++++++++++++++++++----------- src/utilities/DocumentCleaner.ts | 17 ++- 3 files changed, 155 insertions(+), 84 deletions(-) diff --git a/src/include/FrontMatterManager.ts b/src/include/FrontMatterManager.ts index 2d4cdc9..94e8664 100644 --- a/src/include/FrontMatterManager.ts +++ b/src/include/FrontMatterManager.ts @@ -31,7 +31,9 @@ export default class FrontMatterManager { await this.arcana.app.fileManager .processFrontMatter(file, frontmatter => { - tags = parseFrontMatterTags(frontmatter); + tags = + parseFrontMatterTags(frontmatter)?.map(tag => tag.replace('#', '')) ?? + null; }) .catch(e => {}); diff --git a/src/plugins/Darwin/Darwin.ts b/src/plugins/Darwin/Darwin.ts index 0319a8f..1a739af 100644 --- a/src/plugins/Darwin/Darwin.ts +++ b/src/plugins/Darwin/Darwin.ts @@ -14,24 +14,23 @@ import ArcanaPluginBase from 'src/components/ArcanaPluginBase'; import FrontMatterManager from 'src/include/FrontMatterManager'; enum TagStyle { - None, - PascalCase, - CamelCase, - SnakeCase, - KebabCase, - TitleCase, + None = 'None', + PascalCase = 'PascalCase', + CamelCase = 'CamelCase', + SnakeCase = 'SnakeCase', + KebabCase = 'KebabCase', } type DarwinSettings = { minimum_tag_count_to_present: number; - only_show_existing_tags: boolean; + only_suggest_existing_tags: boolean; max_tags_to_show: number; tag_style: TagStyle; }; const DEFAULT_SETTINGS: DarwinSettings = { minimum_tag_count_to_present: 1, - only_show_existing_tags: false, + only_suggest_existing_tags: false, max_tags_to_show: 4, tag_style: TagStyle.None, }; @@ -111,24 +110,62 @@ export default class DarwinPlugin extends ArcanaPluginBase { return true; } + private enforceTagStyle(tag: string): string { + switch (this.setting.tag_style) { + case TagStyle.None: + return tag; + case TagStyle.PascalCase: + return tag + .split(/[-_/]/) + .map(word => word[0].toUpperCase() + word.slice(1)) + .join(''); + case TagStyle.CamelCase: + return tag + .split(/[-_/]/) + .map((word, index) => + index === 0 + ? word[0].toLowerCase() + word.slice(1) + : word[0].toUpperCase() + word.slice(1) + ) + .join(''); + case TagStyle.SnakeCase: + return tag.toLowerCase().replace(/[-_/]/g, '_'); + case TagStyle.KebabCase: + return tag.toLowerCase().replace(/[-_/]/g, '-'); + } + } + + // Robust function to get tags from response private getAdditionalTagsFromResponse( response: string, tagsFromFile: string[] ): string[] { - const removeHash = (tag: string) => tag.replace('#', ''); + // If the response has a colon in it, split and take the second half + if (response.includes(':')) { + response = response.split(':')[1]; + } const tags = response + .replace(/#|,/g, '') .split(' ') - .map(tag => tag.trim()) - .map(removeHash) + .map(tag => { + return tag.trim(); + }) .filter(tag => tag.length > 0) .filter(tag => this.isValidTag(tag)) .filter(tag => !tagsFromFile.includes(tag)) - .filter(tag => this.tagCountCache?.map(t => t[0]).includes(tag)); + .filter(tag => { + if (this.setting.only_suggest_existing_tags) + return this.tagCountCache?.map(t => t[0]).includes(tag); + else return true; + }) + .map(tag => { + if (this.setting.only_suggest_existing_tags) return tag; + else return this.enforceTagStyle(tag); + }); + const uniqueTags = [...new Set(tags)]; - if (uniqueTags.length === 0) return []; - if (uniqueTags.length > this.setting.max_tags_to_show) return []; - return uniqueTags; + return uniqueTags.slice(0, this.setting.max_tags_to_show); } private async fileHasTags(file: TFile): Promise { @@ -142,53 +179,54 @@ export default class DarwinPlugin extends ArcanaPluginBase { this.arcana.settings.PluginSettings['Darwin'] ?? DEFAULT_SETTINGS; // Get lits of all tags periodically - await this.getAllTagsInVault(); this.arcana.registerInterval( window.setInterval(async () => { await this.getAllTagsInVault(); }, 1000 * 45) ); - this.arcana.addCommand({ - id: 'darwin', - name: 'Darwin Tag', - editorCallback: async (editor: Editor, view: MarkdownView) => { - const file = view.file; - await this.autoTagFile(file); - }, - }); + await this.getAllTagsInVault().then(() => { + this.arcana.addCommand({ + id: 'darwin', + name: 'Darwin Tag', + editorCallback: async (editor: Editor, view: MarkdownView) => { + const file = view.file; + await this.autoTagFile(file); + }, + }); - this.arcana.registerEvent( - this.arcana.app.workspace.on( - 'file-menu', - async (menu, tfile: TAbstractFile) => { - if (tfile instanceof TFile) { - menu.addItem(item => { - item.setTitle('Darwin: Tag File'); - item.setIcon('tag'); - item.onClick(async () => { - await this.autoTagFile(tfile); + this.arcana.registerEvent( + this.arcana.app.workspace.on( + 'file-menu', + async (menu, tfile: TAbstractFile) => { + if (tfile instanceof TFile) { + menu.addItem(item => { + item.setTitle('Darwin: Tag File'); + item.setIcon('tag'); + item.onClick(async () => { + await this.autoTagFile(tfile); + }); }); - }); - } else if (tfile instanceof TFolder) { - menu.addItem(item => { - item.setTitle('Darwin: Tag all untagged files in folder'); - item.setIcon('tag'); - item.onClick(async () => { - const folderToTag = tfile; - for (const file of this.arcana.app.vault.getMarkdownFiles()) { - if (file.parent && file.parent.path == folderToTag.path) { - // Need to these synchronously to avoid rate limit - if (await this.fileHasTags(file)) continue; - await this.autoTagFile(file); + } else if (tfile instanceof TFolder) { + menu.addItem(item => { + item.setTitle('Darwin: Tag all untagged files in folder'); + item.setIcon('tag'); + item.onClick(async () => { + const folderToTag = tfile; + for (const file of this.arcana.app.vault.getMarkdownFiles()) { + if (file.parent && file.parent.path == folderToTag.path) { + // Need to these synchronously to avoid rate limit + if (await this.fileHasTags(file)) continue; + await this.autoTagFile(file); + } } - } + }); }); - }); + } } - } - ) - ); + ) + ); + }); } public addSettings(containerEl: HTMLElement) { @@ -220,9 +258,9 @@ export default class DarwinPlugin extends ArcanaPluginBase { ) .addToggle(toggle => { toggle - .setValue(this.setting.only_show_existing_tags) + .setValue(this.setting.only_suggest_existing_tags) .onChange(async (value: boolean) => { - this.setting.only_show_existing_tags = value; + this.setting.only_suggest_existing_tags = value; this.arcana.settings.PluginSettings['Darwin'] = this.setting; await this.arcana.saveSettings(); }); @@ -261,13 +299,14 @@ export default class DarwinPlugin extends ArcanaPluginBase { .addOption('SnakeCase', 'snake_case') .addOption('CamelCase', 'camelCase') .addOption('PascalCase', 'PascalCase') - .setValue(this.setting.tag_style.toString()) + .setValue(this.setting.tag_style) .onChange(async (value: string) => { // Parse the value as a TagStyle const tagStyle = TagStyle[value as keyof typeof TagStyle]; this.setting.tag_style = tagStyle; this.arcana.settings.PluginSettings['Darwin'] = this.setting; await this.arcana.saveSettings(); + console.log(this.setting); }); }); } @@ -278,34 +317,46 @@ export default class DarwinPlugin extends ArcanaPluginBase { const fmm = new FrontMatterManager(this.arcana); const tagsInFrontMatter = await fmm.getTags(file); - const tagsToAdd = await this.askDarwin(file); - if (tagsToAdd.length === 0) { - new Notice('Darwin: Failed to get a good list of tags added'); + const complexResponse = await this.askDarwin(file); + const tagsToAdd = this.getAdditionalTagsFromResponse( + complexResponse, + tagsInFrontMatter + ); + + if (tagsToAdd.length == 0) { + new Notice( + `Darwin: Failed to get a good list of tags added: Got ${tagsToAdd}` + ); return; } - + console.log(`Darwin: Adding tags ${tagsToAdd} to file ${file.path}`); await fmm.setTags(file, tagsInFrontMatter.concat(tagsToAdd)); } - private async askDarwin(file: TFile): Promise { - const purpose = `You are an AI designed to select additional tags to add to a given file. You select from handful of EXISTING TAGS that you will suggest should be added.`; + private DarwinComplexContext(): string { + let purpose = `You are an AI designed to select additional tags to add to a given file.`; + if (this.setting.only_suggest_existing_tags) + purpose += `You select from handful of EXISTING TAGS that you will suggest should be added.`; const existing_tags = `${this.tagCountCache - ?.map(([tag, count]) => `${tag}`) + ?.filter( + ([tag, count]) => count >= this.setting.minimum_tag_count_to_present + ) + .map(([tag, count]) => `${tag}`) .join(' ')}`; const tag_format = `Tags must use the only the following characters: Alphabetical letters, Numbers, Underscore (_), Hyphen (-), Forward slash (/) for Nested tags. Tags cannot contain blank spaces. Tags must contain at least one non-numerical character. For example, #1984 isn't a valid tag, but #y1984 is.`; - const tag_style = `Availabe tagging styles: camelCase, lower kebab-case, snake_case, PascalCase.`; //If the existing tags have a common style, use that as the style of your tags.`; + let rules = `- Return a list of additional tags to add that are most relevant to the file. The list should be space seperated. For example: 'tag1 tag2 tag3'. + - You should only return the list of additional tags and nothing else. Do not give any preamble or other text. + - The length of the list of additional tags must be less than or equal to ${this.setting.max_tags_to_show}. + - The list of additional tags must not contain any tags that are already present in the file. + - Tags must be be valid according to the tag format.`; - const rules = `1. Return a list of additional tags to add that are most relevant to the file. The list should be space seperated. For example: 'tag1 tag2 tag3'. - 2. You should only return the list of additional tags and nothing else. Do not give any preamble or other text. - 3. The length of the list of additional tags must be less than or equal to ${this.setting.max_tags_to_show}. - 4. The list of additional tags must not contain any tags that are already present in the file. - 5. Tags must be be valid according to the tag format. - 6. Only suggest tags that are in the EXISTING TAGS list.`; + if (this.setting.only_suggest_existing_tags) + rules += `\n- Only suggest tags that are in the EXISTING TAGS list.`; - const context = ` + let context = ` [Purpose] ${purpose}. [Tag format] @@ -313,8 +364,14 @@ export default class DarwinPlugin extends ArcanaPluginBase { [Rules] ${rules} `; - // [Tagging Style] - // ${tag_style} + + if (this.setting.only_suggest_existing_tags) { + context += `\n[EXISTING TAGS]\n${existing_tags}`; + } + return context; + } + + private async askDarwin(file: TFile): Promise { const title = file.basename; // Get the document text from the file const documentText = await this.arcana.app.vault.read(file); @@ -324,20 +381,23 @@ export default class DarwinPlugin extends ArcanaPluginBase { const tagsInFile = await this.getTagsForFile(file); let details = ` - [EXISTING TAGS] - ${existing_tags} [Title] ${title} + `; + + if (tagsInFile.length > 0) { + details += `[Tags Present In File] + ${tagsInFile.join(' ')} + `; + } + + details += ` [Text] ${cleanedText} `; - if (tagsInFile.length > 0) - details += `Tags present in file: ${tagsInFile.join(' ')}`; + const context = this.DarwinComplexContext(); const response = await this.arcana.complete(details, context); - console.log(`Details: ${details}`); - console.log(`Response: ${response}`); - - return this.getAdditionalTagsFromResponse(response, tagsInFile); + return response; } } diff --git a/src/utilities/DocumentCleaner.ts b/src/utilities/DocumentCleaner.ts index 986af79..9a2b8e9 100644 --- a/src/utilities/DocumentCleaner.ts +++ b/src/utilities/DocumentCleaner.ts @@ -1,8 +1,17 @@ export function removeFrontMatter(text: string): string { - //Remove the front matter if it exists - const frontMatterRegex = /^---\n[\s\S]*\n---\n/; - text = text.replace(frontMatterRegex, ''); - return text; + // Check if the text has front matter + if (!text.startsWith('---')) { + return text; + } + // Remove the front matter from just the start of the text + const lines = text.split('\n'); + let i = 1; + for (; i < lines.length; i++) { + if (lines[i].startsWith('---')) { + break; + } + } + return lines.slice(i + 1).join('\n'); } export function surroundWithMarkdown(text: string): string { From dd9bd5391d9dbc97229cf695c9fe1fd8e067b7b9 Mon Sep 17 00:00:00 2001 From: A-F-V Date: Wed, 5 Jul 2023 23:36:37 +0100 Subject: [PATCH 4/4] Update Darwin.ts --- src/plugins/Darwin/Darwin.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/Darwin/Darwin.ts b/src/plugins/Darwin/Darwin.ts index 1a739af..a0915b2 100644 --- a/src/plugins/Darwin/Darwin.ts +++ b/src/plugins/Darwin/Darwin.ts @@ -178,11 +178,12 @@ export default class DarwinPlugin extends ArcanaPluginBase { this.setting = this.arcana.settings.PluginSettings['Darwin'] ?? DEFAULT_SETTINGS; - // Get lits of all tags periodically + // Get lists of all tags periodically + // TODO: Just add tags heuristicly after initial load this.arcana.registerInterval( window.setInterval(async () => { await this.getAllTagsInVault(); - }, 1000 * 45) + }, 1000 * 60) ); await this.getAllTagsInVault().then(() => {