From 66264d38c416e6c4c79d227b5bd6fb5ef9ad4b70 Mon Sep 17 00:00:00 2001 From: henk717 Date: Tue, 28 Mar 2023 00:23:10 +0200 Subject: [PATCH 01/35] Add Mixes --- colab/GPU.ipynb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/colab/GPU.ipynb b/colab/GPU.ipynb index 5e0888099..7d9fbb5f4 100644 --- a/colab/GPU.ipynb +++ b/colab/GPU.ipynb @@ -82,7 +82,7 @@ "#@title <-- Select your model below and then click this to start KoboldAI\n", "#@markdown You can find a description of the models below along with instructions on how to start KoboldAI.\n", "\n", - "Model = \"Nerys V2 6B\" #@param [\"Nerys V2 6B\", \"Erebus 6B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Pygmalion 6B\", \"Pygmalion 6B Dev\", \"Lit V2 6B\", \"Lit 6B\", \"Shinen 6B\", \"Nerys 2.7B\", \"AID 2.7B\", \"Erebus 2.7B\", \"Janeway 2.7B\", \"Picard 2.7B\", \"Horni LN 2.7B\", \"Horni 2.7B\", \"Shinen 2.7B\", \"OPT 2.7B\", \"Fairseq Dense 2.7B\", \"Neo 2.7B\"] {allow-input: true}\n", + "Model = \"Nerys V2 6B\" #@param [\"Nerys V2 6B\", \"Nerybus 6B\", \"Erebus 6B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"PPO_Pygway 6B\", \"Pygmalion 6B\", \"Pygmalion 6B Dev\", \"Lit V2 6B\", \"Lit 6B\", \"Shinen 6B\", \"Nerys 2.7B\", \"AID 2.7B\", \"Erebus 2.7B\", \"Janeway 2.7B\", \"Picard 2.7B\", \"Horni LN 2.7B\", \"Horni 2.7B\", \"Shinen 2.7B\", \"OPT 2.7B\", \"Fairseq Dense 2.7B\", \"Neo 2.7B\"] {allow-input: true}\n", "Version = \"Official\" #@param [\"Official\", \"United\"] {allow-input: true}\n", "Provider = \"Cloudflare\" #@param [\"Localtunnel\", \"Cloudflare\"]\n", "use_google_drive = True #@param {type:\"boolean\"}\n", @@ -104,6 +104,10 @@ " Model = \"KoboldAI/OPT-6B-nerys-v2\"\n", " path = \"\"\n", " download = \"\"\n", + "elif Model == \"Nerybus 6B\":\n", + " Model = \"KoboldAI/OPT-6.7B-Nerybus-Mix\"\n", + " path = \"\"\n", + " download = \"\"\n", "elif Model == \"Erebus 6B\":\n", " Model = \"KoboldAI/OPT-6.7B-Erebus\"\n", " path = \"\"\n", @@ -120,6 +124,11 @@ " Model = \"KoboldAI/GPT-J-6B-Adventure\"\n", " path = \"\"\n", " download = \"\"\n", + "elif Model == \"PPO_Pygway 6B\":\n", + " Model = \"KoboldAI/PPO_Pygway-6b-Mix\"\n", + " path = \"\"\n", + " download = \"\"\n", + " Version = \"United\"\n", "elif Model == \"Pygmalion 6B\":\n", " Model = \"PygmalionAI/pygmalion-6b\"\n", " path = \"\"\n", From f826930c0242e1d38f7473fb20a892f8e0ea2801 Mon Sep 17 00:00:00 2001 From: YellowRoseCx <80486540+YellowRoseCx@users.noreply.github.com> Date: Wed, 29 Mar 2023 17:41:01 -0500 Subject: [PATCH 02/35] Update GPU.ipynb removed litv2-6B-rev3 --- colab/GPU.ipynb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/colab/GPU.ipynb b/colab/GPU.ipynb index 7d9fbb5f4..c22eb8b6b 100644 --- a/colab/GPU.ipynb +++ b/colab/GPU.ipynb @@ -140,10 +140,6 @@ " path = \"\"\n", " Version = \"United\"\n", " download = \"\"\n", - "elif Model == \"Lit V2 6B\":\n", - " Model = \"hakurei/litv2-6B-rev3\"\n", - " path = \"\"\n", - " download = \"\"\n", "elif Model == \"Lit 6B\":\n", " Model = \"hakurei/lit-6B\"\n", " path = \"\"\n", @@ -237,7 +233,7 @@ "| [Skein](https://huggingface.co/KoboldAI/GPT-J-6B-Skein) by VE\\_FORBRYDERNE | Adventure | Skein is best used with Adventure mode enabled, it consists of a 4 times larger adventure dataset than the Adventure model making it excellent for text adventure gaming. On top of that it also consists of light novel training further expanding its knowledge and writing capabilities. It can be used with the You filter bias if you wish to write Novels with it, but dedicated Novel models can perform better for this task. |\n", "| [Adventure](https://huggingface.co/KoboldAI/GPT-J-6B-Adventure) by VE\\_FORBRYDERNE | Adventure | Adventure is a 6B model designed to mimick the behavior of AI Dungeon. It is exclusively for Adventure Mode and can take you on the epic and wackey adventures that AI Dungeon players love. It also features the many tropes of AI Dungeon as it has been trained on very similar data. It must be used in second person (You). |\n", "| [Pygmalion](https://huggingface.co/KoboldAI/GPT-J-6B-Adventure) by PygmalionAI | Chatbot | Pygmalion is a chat model that has been based on a few models that came before it. First the model originates from LitV2, it was then trained by Haru on a chat dataset to create ConvoGPT. ConvoGPT was then trained by PygmalionAI on chat data that contains longer responses and emotions. Making for a higher quality chat experience than you can get from other models such as Erebus that are not directly trained on chatting. |\n", - "| [Lit](https://huggingface.co/hakurei/lit-6B) ([V2](https://huggingface.co/hakurei/litv2-6B-rev3)) by Haru | NSFW | Lit is a great NSFW model trained by Haru on both a large set of Literotica stories and high quality novels along with tagging support. Creating a high quality model for your NSFW stories. This model is exclusively a novel model and is best used in third person. |\n", + "| [Lit](https://huggingface.co/hakurei/lit-6B) by Haru | NSFW | Lit is a great NSFW model trained by Haru on both a large set of Literotica stories and high quality novels along with tagging support. Creating a high quality model for your NSFW stories. This model is exclusively a novel model and is best used in third person. |\n", "| [OPT](https://huggingface.co/facebook/opt-13b) by Metaseq | Generic | OPT is considered one of the best base models as far as content goes, its behavior has the strengths of both GPT-Neo and Fairseq Dense. Compared to Neo duplicate and unnecessary content has been left out, while additional literature was added in similar to the Fairseq Dense model. The Fairseq Dense model however lacks the broader data that OPT does have. The biggest downfall of OPT is its license, which prohibits any commercial usage, or usage beyond research purposes. |\n", "| [Neo(X)](https://huggingface.co/EleutherAI/gpt-neox-20b) by EleutherAI | Generic | NeoX is the largest EleutherAI model currently available, being a generic model it is not particularly trained towards anything and can do a variety of writing, Q&A and coding tasks. 20B's performance is closely compared to the 13B models and it is worth trying both especially if you have a task that does not involve english writing. Its behavior will be similar to the GPT-J-6B model since they are trained on the same dataset but with more sensitivity towards repetition penalty and with more knowledge. |\n", "| [Fairseq Dense](https://huggingface.co/KoboldAI/fairseq-dense-13B) | Generic | Trained by Facebook Researchers this model stems from the MOE research project within Fairseq. This particular version has been converted by us for use in KoboldAI. It is known to be on par with the larger 20B model from EleutherAI and considered as better for pop culture and language tasks. Because the model has never seen a new line (enter) it may perform worse on formatting and paragraphing. Compared to other models the dataset focuses primarily on literature and contains little else. |\n", @@ -268,4 +264,4 @@ } } ] -} \ No newline at end of file +} From 3c48a77a52b42ae2d163338745848e54fcabb7a5 Mon Sep 17 00:00:00 2001 From: YellowRoseCx <80486540+YellowRoseCx@users.noreply.github.com> Date: Wed, 29 Mar 2023 17:44:44 -0500 Subject: [PATCH 03/35] Update README.md changed Colab GPU models listed to their higher quality counter parts --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 20a1957a4..789b78d1b 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ If you would like to play KoboldAI online for free on a powerful computer you ca Each edition features different models and requires different hardware to run, this means that if you are unable to obtain a TPU or a GPU you might still be able to use the other version. The models you can use are listed underneath the edition. To open a Colab click the big link featuring the editions name. -## [TPU Edition Model Descriptions](https://colab.research.google.com/github/KoboldAI/KoboldAI-Client/blob/main/colab/TPU.ipynb) +## [Models the TPU can run:](https://colab.research.google.com/github/KoboldAI/KoboldAI-Client/blob/main/colab/TPU.ipynb) | Model | Style | Description | | --- | --- | --- | @@ -64,21 +64,21 @@ Each edition features different models and requires different hardware to run, t | [Fairseq Dense](https://huggingface.co/KoboldAI/fairseq-dense-13B) | Generic | Trained by Facebook Researchers this model stems from the MOE research project within Fairseq. This particular version has been converted by us for use in KoboldAI. It is known to be on par with the larger 20B model from EleutherAI and considered as better for pop culture and language tasks. Because the model has never seen a new line (enter) it may perform worse on formatting and paragraphing. Compared to other models the dataset focuses primarily on literature and contains little else. | | [GPT-J-6B](https://huggingface.co/EleutherAI/gpt-j-6B) by EleutherAI | Generic | This model serves as the basis for most other 6B models (Some being based on Fairseq Dense instead). Being trained on the Pile and not biased towards anything in particular it is suitable for a variety of tasks such as writing, Q&A and coding tasks. You will likely get better result with larger generic models or finetuned models. | -## [GPU Edition Model Descriptions](https://colab.research.google.com/github/KoboldAI/KoboldAI-Client/blob/main/colab/GPU.ipynb) +## [Models the Colab GPU can run:](https://colab.research.google.com/github/KoboldAI/KoboldAI-Client/blob/main/colab/GPU.ipynb) | Model | Style | Description | | --- | --- | --- | -| [Nerys](https://huggingface.co/KoboldAI/fairseq-dense-2.7B-Nerys) by Mr Seeker | Novel/Adventure | Nerys is a hybrid model based on Pike (A newer Janeway), on top of the Pike dataset you also get some Light Novels, Adventure mode support and a little bit of Shinen thrown in the mix. The end result is a very diverse model that is heavily biased towards SFW novel writing, but one that can go beyond its novel training and make for an excellent adventure model to. Adventure mode is best played from a second person perspective, but can be played in first or third person as well. Novel writing can be done best from the first or third person. | -| [Erebus](https://huggingface.co/KoboldAI/OPT-2.7B-Erebus) by Mr Seeker | NSFW | Erebus is our community's flagship NSFW model, being a combination of multiple large datasets that include Literotica, Shinen and erotic novels from Nerys and featuring thourough tagging support it covers the vast majority of erotic writing styles. This model is capable of replacing both the Lit and Shinen models in terms of content and style and has been well received as (one of) the best NSFW models out there. If you wish to use this model for commercial or non research usage we recommend choosing the 20B version as that one is not subject to the restrictive OPT license. | -| [Janeway](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-Janeway) by Mr Seeker | Novel | Janeway is a model created from Picard's dataset combined with a brand new collection of ebooks. This model is trained on 20% more content than Picard and has been trained on literature from various genres. Although the model is mainly focussed on SFW, romantic scenes might involve a degree of nudity. | -| [Picard](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-Picard) by Mr Seeker | Novel | Picard is a model trained for SFW Novels based on Neo 2.7B. It is focused on Novel style writing without the NSFW bias. While the name suggests a sci-fi model this model is designed for Novels of a variety of genre's. It is meant to be used in KoboldAI's regular mode. | -| [AID](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-AID) by melastacho | Adventure | Also know as Adventure 2.7B this is a clone of the AI Dungeon Classic model and is best known for the epic wackey adventures that AI Dungeon Classic players love. | -| [Horni LN](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-Horni-LN) by finetune | Novel | This model is based on Horni 2.7B and retains its NSFW knowledge, but was then further biased towards SFW novel stories. If you seek a balance between a SFW Novel model and a NSFW model this model should be a good choice. | -| [Horni](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-Horni) by finetune | NSFW | This model is tuned on Literotica to produce a Novel style model biased towards NSFW content. Can still be used for SFW stories but will have a bias towards NSFW content. It is meant to be used in KoboldAI's regular mode. | -| [Shinen](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-Shinen) by Mr Seeker | NSFW | Shinen is an alternative to the Horni model designed to be more explicit. If Horni is to tame for you Shinen might produce better results. While it is a Novel model it is unsuitable for SFW stories due to its heavy NSFW bias. Shinen will not hold back. It is meant to be used in KoboldAI's regular mode. | -| [OPT](https://huggingface.co/facebook/opt-2.7b) by Metaseq | Generic | OPT is considered one of the best base models as far as content goes, its behavior has the strengths of both GPT-Neo and Fairseq Dense. Compared to Neo duplicate and unnecessary content has been left out, while additional literature was added in similar to the Fairseq Dense model. The Fairseq Dense model however lacks the broader data that OPT does have. The biggest downfall of OPT is its license, which prohibits any commercial usage, or usage beyond research purposes. | -| [Fairseq Dense](https://huggingface.co/KoboldAI/fairseq-dense-2.7B) | Generic | Trained by Facebook Researchers this model stems from the MOE research project within Fairseq. This particular version has been converted by us for use in KoboldAI. It is known to be on par with the larger models from EleutherAI and considered as better for pop culture and language tasks. Because the model has never seen a new line (enter) it may perform worse on formatting and paragraphing. Compared to other models the dataset focuses primarily on literature and contains little else. | -| [Neo](https://huggingface.co/EleutherAI/gpt-neo-2.7B) by EleutherAI | Generic | This is the base model for all the other 2.7B models, it is best used when you have a use case that we have no other models available for, such as writing blog articles or programming. It can also be a good basis for the experience of some of the softprompts if your softprompt is not about a subject the other models cover. | +| [Pygmalion-6b](https://huggingface.co/PygmalionAI/pygmalion-6b) by Pygmalion AI | NSFW/Chat | Pymalion 6B is a proof-of-concept dialogue model based on EleutherAI's GPT-J-6B. Warning: This model is NOT suitable for use by minors. It will output X-rated content under certain circumstances. The fine-tuning dataset consisted of 56MB of dialogue data gathered from multiple sources, which includes both real and partially machine-generated conversations. | +| [Nerys-6b](https://huggingface.co/KoboldAI/OPT-6B-nerys-v2) by Mr Seeker | Novel/Adventure | Nerys is a hybrid model based on Pike (A newer Janeway), on top of the Pike dataset you also get some Light Novels, Adventure mode support and a little bit of Shinen thrown in the mix. The end result is a very diverse model that is heavily biased towards SFW novel writing, but one that can go beyond its novel training and make for an excellent adventure model to. Adventure mode is best played from a second person perspective, but can be played in first or third person as well. Novel writing can be done best from the first or third person. | +| [Erebus-6.7b](https://huggingface.co/KoboldAI/OPT-6.7B-Erebus) by Mr Seeker | NSFW | Erebus is our community's flagship NSFW model, being a combination of multiple large datasets that include Literotica, Shinen and erotic novels from Nerys and featuring thourough tagging support it covers the vast majority of erotic writing styles. This model is capable of replacing both the Lit and Shinen models in terms of content and style and has been well received as (one of) the best NSFW models out there. If you wish to use this model for commercial or non research usage we recommend choosing the 20B version as that one is not subject to the restrictive OPT license. | +| [Skein-6b](https://huggingface.co/KoboldAI/GPT-J-6B-Skein) by Mr Seeker | Adventure | This model is designed for creative story generation. It can understand both free-form text and text written in interactive fiction style with actions starting with "> You". Trained with light novels and assorted interactive fiction. | +| [Adventure 6b](https://huggingface.co/KoboldAI/GPT-J-6B-Adventure) by Mr Seeker | Adventure | This is a clone of the AI Dungeon Classic model and is best known for the epic wackey adventures that AI Dungeon Classic players love. | +| [PPO-Pygway-6b](https://huggingface.co/KoboldAI/PPO_Pygway-6b-Mix) by TeH_Venom | Instruct-tuned Chat | PPO-Pygway is a model that merges together KoboldAI/GPT-J-6B-Janeway, reciprocate/ppo_hh_gpt-j, and Pygmalion/Pygmalion-6b; all three models were blended in a two step process using a simple weighted parameter method. This model may generate NSFW content. | +| [Janeway-6b](https://huggingface.co/KoboldAI/GPT-J-6B-Janeway) by Mr Seeker | Novel | GPT-J 6B-Janeway is a finetune created using EleutherAI's GPT-J 6B model. The training data contains around 2210 ebooks, mostly in the sci-fi and fantasy genres. The dataset is based on the same dataset used by GPT-Neo-2.7B-Picard, with 20% more data in various genres.| +| [Lit-6b](https://huggingface.co/hakurei/lit-6B) by Hakurei | Novel | Lit-6B is a GPT-J 6B model fine-tuned on 2GB of a diverse range of light novels, erotica, and annotated literature for the purpose of generating novel-like fictional text. The model used for fine-tuning is GPT-J, which is a 6 billion parameter auto-regressive language model trained on The Pile. | +| [Nerybus-6.7b](https://huggingface.co/KoboldAI/OPT-6.7B-Nerybus-Mix) by Concedo | Novel/NSFW | This model is based on OPT-6.7b-Erebus and Merged with Nerys retaining its NSFW knowledge, but was then further biased towards SFW novel stories. If you seek a balance between a SFW Novel model and a NSFW model this model should be a good choice. | +| [Shinen-6b](https://huggingface.co/KoboldAI/GPT-J-6B-Shinen) by Mr Seeker | Novel/NSFW | Shinen is an alternative to the OPT-license based Erebus model. While it is a Novel model it is unsuitable for SFW stories due to its heavy NSFW bias. Shinen will not hold back. It is meant to be used in KoboldAI's regular mode. | +| [Various 2.7b models]() by various | Various smaller models are also possible to load in GPU colab. | | ### Styles From 72b4669563df2fd879aa54e12cbb8aa2e37fb104 Mon Sep 17 00:00:00 2001 From: henk717 Date: Thu, 30 Mar 2023 23:41:35 +0200 Subject: [PATCH 04/35] Fix the chex dependency --- requirements_mtj.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_mtj.txt b/requirements_mtj.txt index e66e1ba7f..9447541f9 100644 --- a/requirements_mtj.txt +++ b/requirements_mtj.txt @@ -5,6 +5,7 @@ requests dm-haiku == 0.0.5 jax == 0.2.21 jaxlib >= 0.1.69, <= 0.3.7 +chex == 0.1.5 transformers == 4.24.0 huggingface_hub==0.12.1 progressbar2 From 4a8b099888e8a00bc7b66aea48ed4053166ceaf6 Mon Sep 17 00:00:00 2001 From: Henk Date: Sun, 2 Apr 2023 00:29:56 +0200 Subject: [PATCH 05/35] Model loading fix --- aiserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiserver.py b/aiserver.py index 3368447b1..7cbbc8aca 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1123,7 +1123,7 @@ def move_model_to_devices(model): for key, value in model.state_dict().items(): target_dtype = torch.float32 if breakmodel.primary_device == "cpu" else torch.float16 if(value.dtype is not target_dtype): - accelerate.utils.set_module_tensor_to_device(model, key, target_dtype) + accelerate.utils.set_module_tensor_to_device(model, key, torch.device(value.device), value, target_dtype) disk_blocks = breakmodel.disk_blocks gpu_blocks = breakmodel.gpu_blocks ram_blocks = len(utils.layers_module_names) - sum(gpu_blocks) From 8bf533da9a32f7a46825988681bb270b196ff965 Mon Sep 17 00:00:00 2001 From: Henk Date: Tue, 4 Apr 2023 01:47:59 +0200 Subject: [PATCH 06/35] Pin Accelerate Version --- environments/huggingface.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/huggingface.yml b/environments/huggingface.yml index 69e0a49e7..2eb1bfe0c 100644 --- a/environments/huggingface.yml +++ b/environments/huggingface.yml @@ -33,7 +33,7 @@ dependencies: - transformers==4.25.1 - huggingface_hub==0.12.1 - safetensors - - accelerate + - accelerate==0.18.0 - git+https://github.com/VE-FORBRYDERNE/mkultra - flask-session - python-socketio[client] From 4b71da17147e76f4a9ceb394f7863945263bdbb0 Mon Sep 17 00:00:00 2001 From: Henk Date: Tue, 4 Apr 2023 17:20:43 +0200 Subject: [PATCH 07/35] Horde settings in the UI --- gensettings.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/gensettings.py b/gensettings.py index 863a2ba57..8d68b4b53 100644 --- a/gensettings.py +++ b/gensettings.py @@ -816,6 +816,40 @@ "ui_level": 2 }, { + "UI_V2_Only": True, + "uitype": "text", + "unit": "text", + "label": "Horde Worker Name", + "id": "horde_worker_name", + "min": 0, + "max": 1, + "step": 1, + "default": 0, + "tooltip": "This is the name of your worker that shows up on the Horde", + "menu_path": "Settings", + "sub_path": "Other", + "classname": "user", + "name": "horde_worker_name", + "ui_level": 2 + }, + { + "UI_V2_Only": True, + "uitype": "password", + "unit": "text", + "label": "Horde API Key", + "id": "horde_api_key", + "min": 0, + "max": 1, + "step": 1, + "default": 0, + "tooltip": "You can paste your API key here for faster generations and to earn Kudo's as a worker", + "menu_path": "Settings", + "sub_path": "Other", + "classname": "user", + "name": "horde_api_key", + "ui_level": 2 + }, + { "UI_V2_Only": True, "uitype": "toggle", "unit": "bool", From bb51127bbfd41ba8172c17231d0b418209c8619e Mon Sep 17 00:00:00 2001 From: henk717 Date: Tue, 4 Apr 2023 19:37:15 +0200 Subject: [PATCH 08/35] We no longer support Pygmalion on Colab due to Google's Pygmalion ban --- colab/GPU.ipynb | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/colab/GPU.ipynb b/colab/GPU.ipynb index c22eb8b6b..517d06613 100644 --- a/colab/GPU.ipynb +++ b/colab/GPU.ipynb @@ -82,7 +82,7 @@ "#@title <-- Select your model below and then click this to start KoboldAI\n", "#@markdown You can find a description of the models below along with instructions on how to start KoboldAI.\n", "\n", - "Model = \"Nerys V2 6B\" #@param [\"Nerys V2 6B\", \"Nerybus 6B\", \"Erebus 6B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"PPO_Pygway 6B\", \"Pygmalion 6B\", \"Pygmalion 6B Dev\", \"Lit V2 6B\", \"Lit 6B\", \"Shinen 6B\", \"Nerys 2.7B\", \"AID 2.7B\", \"Erebus 2.7B\", \"Janeway 2.7B\", \"Picard 2.7B\", \"Horni LN 2.7B\", \"Horni 2.7B\", \"Shinen 2.7B\", \"OPT 2.7B\", \"Fairseq Dense 2.7B\", \"Neo 2.7B\"] {allow-input: true}\n", + "Model = \"Nerys V2 6B\" #@param [\"Nerys V2 6B\", \"Nerybus 6B\", \"Erebus 6B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"PPO_Pygway 6B\", \"Lit V2 6B\", \"Lit 6B\", \"Shinen 6B\", \"Nerys 2.7B\", \"AID 2.7B\", \"Erebus 2.7B\", \"Janeway 2.7B\", \"Picard 2.7B\", \"Horni LN 2.7B\", \"Horni 2.7B\", \"Shinen 2.7B\", \"OPT 2.7B\", \"Fairseq Dense 2.7B\", \"Neo 2.7B\"] {allow-input: true}\n", "Version = \"Official\" #@param [\"Official\", \"United\"] {allow-input: true}\n", "Provider = \"Cloudflare\" #@param [\"Localtunnel\", \"Cloudflare\"]\n", "use_google_drive = True #@param {type:\"boolean\"}\n", @@ -129,17 +129,6 @@ " path = \"\"\n", " download = \"\"\n", " Version = \"United\"\n", - "elif Model == \"Pygmalion 6B\":\n", - " Model = \"PygmalionAI/pygmalion-6b\"\n", - " path = \"\"\n", - " download = \"\"\n", - " Version = \"United\"\n", - "elif Model == \"Pygmalion 6B Dev\":\n", - " Model = \"PygmalionAI/pygmalion-6b\"\n", - " Revision = \"--revision dev\"\n", - " path = \"\"\n", - " Version = \"United\"\n", - " download = \"\"\n", "elif Model == \"Lit 6B\":\n", " Model = \"hakurei/lit-6B\"\n", " path = \"\"\n", @@ -264,4 +253,4 @@ } } ] -} +} \ No newline at end of file From fd12214091d33af78c66decf540cde239a798cfd Mon Sep 17 00:00:00 2001 From: henk717 Date: Tue, 4 Apr 2023 19:40:22 +0200 Subject: [PATCH 09/35] Clean the description of the GPU colab --- colab/GPU.ipynb | 2 -- 1 file changed, 2 deletions(-) diff --git a/colab/GPU.ipynb b/colab/GPU.ipynb index 517d06613..c6c1e5ba6 100644 --- a/colab/GPU.ipynb +++ b/colab/GPU.ipynb @@ -203,7 +203,6 @@ "| [Janeway](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-Janeway) by Mr Seeker | Novel | Janeway is a model created from Picard's dataset combined with a brand new collection of ebooks. This model is trained on 20% more content than Picard and has been trained on literature from various genres. Although the model is mainly focussed on SFW, romantic scenes might involve a degree of nudity. |\n", "| [Picard](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-Picard) by Mr Seeker | Novel | Picard is a model trained for SFW Novels based on Neo 2.7B. It is focused on Novel style writing without the NSFW bias. While the name suggests a sci-fi model this model is designed for Novels of a variety of genre's. It is meant to be used in KoboldAI's regular mode. |\n", "| [AID](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-AID) by melastacho | Adventure | Also know as Adventure 2.7B this is a clone of the AI Dungeon Classic model and is best known for the epic wackey adventures that AI Dungeon Classic players love. |\n", - "| [Pygmalion](https://huggingface.co/KoboldAI/GPT-J-6B-Adventure) by PygmalionAI | Chatbot | Pygmalion is a chat model that has been based on a few models that came before it. First the model originates from LitV2, it was then trained by Haru on a chat dataset to create ConvoGPT. ConvoGPT was then trained by PygmalionAI on chat data that contains longer responses and emotions. Making for a higher quality chat experience than you can get from other models such as Erebus that are not directly trained on chatting. |\n", "| [Horni LN](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-Horni-LN) by finetune | Novel | This model is based on Horni 2.7B and retains its NSFW knowledge, but was then further biased towards SFW novel stories. If you seek a balance between a SFW Novel model and a NSFW model this model should be a good choice. |\n", "| [Horni](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-Horni) by finetune | NSFW | This model is tuned on Literotica to produce a Novel style model biased towards NSFW content. Can still be used for SFW stories but will have a bias towards NSFW content. It is meant to be used in KoboldAI's regular mode. |\n", "| [Shinen](https://huggingface.co/KoboldAI/GPT-Neo-2.7B-Shinen) by Mr Seeker | NSFW | Shinen is an alternative to the Horni model designed to be more explicit. If Horni is to tame for you Shinen might produce better results. While it is a Novel model it is unsuitable for SFW stories due to its heavy NSFW bias. Shinen will not hold back. It is meant to be used in KoboldAI's regular mode. |\n", @@ -221,7 +220,6 @@ "| [Shinen](https://huggingface.co/KoboldAI/fairseq-dense-13B-Shinen) by Mr Seeker | NSFW | Shinen is an NSFW model trained on a variety of stories from the website Sexstories it contains many different kinks. It has been merged into the larger (and better) Erebus model. |\n", "| [Skein](https://huggingface.co/KoboldAI/GPT-J-6B-Skein) by VE\\_FORBRYDERNE | Adventure | Skein is best used with Adventure mode enabled, it consists of a 4 times larger adventure dataset than the Adventure model making it excellent for text adventure gaming. On top of that it also consists of light novel training further expanding its knowledge and writing capabilities. It can be used with the You filter bias if you wish to write Novels with it, but dedicated Novel models can perform better for this task. |\n", "| [Adventure](https://huggingface.co/KoboldAI/GPT-J-6B-Adventure) by VE\\_FORBRYDERNE | Adventure | Adventure is a 6B model designed to mimick the behavior of AI Dungeon. It is exclusively for Adventure Mode and can take you on the epic and wackey adventures that AI Dungeon players love. It also features the many tropes of AI Dungeon as it has been trained on very similar data. It must be used in second person (You). |\n", - "| [Pygmalion](https://huggingface.co/KoboldAI/GPT-J-6B-Adventure) by PygmalionAI | Chatbot | Pygmalion is a chat model that has been based on a few models that came before it. First the model originates from LitV2, it was then trained by Haru on a chat dataset to create ConvoGPT. ConvoGPT was then trained by PygmalionAI on chat data that contains longer responses and emotions. Making for a higher quality chat experience than you can get from other models such as Erebus that are not directly trained on chatting. |\n", "| [Lit](https://huggingface.co/hakurei/lit-6B) by Haru | NSFW | Lit is a great NSFW model trained by Haru on both a large set of Literotica stories and high quality novels along with tagging support. Creating a high quality model for your NSFW stories. This model is exclusively a novel model and is best used in third person. |\n", "| [OPT](https://huggingface.co/facebook/opt-13b) by Metaseq | Generic | OPT is considered one of the best base models as far as content goes, its behavior has the strengths of both GPT-Neo and Fairseq Dense. Compared to Neo duplicate and unnecessary content has been left out, while additional literature was added in similar to the Fairseq Dense model. The Fairseq Dense model however lacks the broader data that OPT does have. The biggest downfall of OPT is its license, which prohibits any commercial usage, or usage beyond research purposes. |\n", "| [Neo(X)](https://huggingface.co/EleutherAI/gpt-neox-20b) by EleutherAI | Generic | NeoX is the largest EleutherAI model currently available, being a generic model it is not particularly trained towards anything and can do a variety of writing, Q&A and coding tasks. 20B's performance is closely compared to the 13B models and it is worth trying both especially if you have a task that does not involve english writing. Its behavior will be similar to the GPT-J-6B model since they are trained on the same dataset but with more sensitivity towards repetition penalty and with more knowledge. |\n", From 29c2d4b7a6f35bad14b52c7e252da9e20e00aa62 Mon Sep 17 00:00:00 2001 From: henk717 Date: Tue, 4 Apr 2023 19:51:18 +0200 Subject: [PATCH 10/35] Removing Pygmalion from the TPU colab to get it unbanned --- colab/TPU.ipynb | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/colab/TPU.ipynb b/colab/TPU.ipynb index e432b958c..48fc7bdc8 100644 --- a/colab/TPU.ipynb +++ b/colab/TPU.ipynb @@ -79,7 +79,7 @@ "#@title <-- Select your model below and then click this to start KoboldAI\n", "#@markdown You can find a description of the models below along with instructions on how to start KoboldAI.\n", "\n", - "Model = \"Nerys 13B V2\" #@param [\"Nerys 13B V2\", \"Nerybus 13B\", \"Erebus 13B\", \"Janeway 13B\", \"Shinen 13B\", \"Skein 20B\", \"Erebus 20B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Shinen 6B\", \"Pygmalion 6B\", \"Pygmalion 6B Dev\", \"Lit V2 6B\", \"Lit 6B\", \"NeoX 20B\", \"OPT 13B\", \"Fairseq Dense 13B\", \"GPT-J-6B\"] {allow-input: true}\n", + "Model = \"Nerys 13B V2\" #@param [\"Nerys 13B V2\", \"Nerybus 13B\", \"Erebus 13B\", \"Janeway 13B\", \"Shinen 13B\", \"Skein 20B\", \"Erebus 20B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Shinen 6B\", \"Lit V2 6B\", \"Lit 6B\", \"NeoX 20B\", \"OPT 13B\", \"Fairseq Dense 13B\", \"GPT-J-6B\"] {allow-input: true}\n", "Version = \"Official\" #@param [\"Official\", \"United\"] {allow-input: true}\n", "Provider = \"Cloudflare\" #@param [\"Localtunnel\", \"Cloudflare\"]\n", "use_google_drive = True #@param {type:\"boolean\"}\n", @@ -148,17 +148,6 @@ " Model = \"KoboldAI/GPT-J-6B-Adventure\"\n", " path = \"\"\n", " download = \"\"\n", - "elif Model == \"Pygmalion 6B\":\n", - " Model = \"PygmalionAI/pygmalion-6b\"\n", - " path = \"\"\n", - " download = \"\"\n", - " Version = \"United\"\n", - "elif Model == \"Pygmalion 6B Dev\":\n", - " Model = \"PygmalionAI/pygmalion-6b\"\n", - " Revision = \"--revision dev\"\n", - " path = \"\"\n", - " Version = \"United\"\n", - " download = \"\"\n", "elif Model == \"Lit V2 6B\":\n", " Model = \"hakurei/litv2-6B-rev3\"\n", " path = \"\"\n", @@ -208,7 +197,6 @@ "| [Shinen](https://huggingface.co/KoboldAI/fairseq-dense-13B-Shinen) by Mr Seeker | NSFW | Shinen is an NSFW model trained on a variety of stories from the website Sexstories it contains many different kinks. It has been merged into the larger (and better) Erebus model. |\n", "| [Skein](https://huggingface.co/KoboldAI/GPT-J-6B-Skein) by VE\\_FORBRYDERNE | Adventure | Skein is best used with Adventure mode enabled, it consists of a 4 times larger adventure dataset than the Adventure model making it excellent for text adventure gaming. On top of that it also consists of light novel training further expanding its knowledge and writing capabilities. It can be used with the You filter bias if you wish to write Novels with it, but dedicated Novel models can perform better for this task. |\n", "| [Adventure](https://huggingface.co/KoboldAI/GPT-J-6B-Adventure) by VE\\_FORBRYDERNE | Adventure | Adventure is a 6B model designed to mimick the behavior of AI Dungeon. It is exclusively for Adventure Mode and can take you on the epic and wackey adventures that AI Dungeon players love. It also features the many tropes of AI Dungeon as it has been trained on very similar data. It must be used in second person (You). |\n", - "| [Pygmalion](https://huggingface.co/KoboldAI/GPT-J-6B-Adventure) by PygmalionAI | Chatbot | Pygmalion is a chat model that has been based on a few models that came before it. First the model originates from LitV2, it was then trained by Haru on a chat dataset to create ConvoGPT. ConvoGPT was then trained by PygmalionAI on chat data that contains longer responses and emotions. Making for a higher quality chat experience than you can get from other models such as Erebus that are not directly trained on chatting. |\n", "| [Lit](https://huggingface.co/hakurei/lit-6B) ([V2](https://huggingface.co/hakurei/litv2-6B-rev3)) by Haru | NSFW | Lit is a great NSFW model trained by Haru on both a large set of Literotica stories and high quality novels along with tagging support. Creating a high quality model for your NSFW stories. This model is exclusively a novel model and is best used in third person. |\n", "| [OPT](https://huggingface.co/facebook/opt-13b) by Metaseq | Generic | OPT is considered one of the best base models as far as content goes, its behavior has the strengths of both GPT-Neo and Fairseq Dense. Compared to Neo duplicate and unnecessary content has been left out, while additional literature was added in similar to the Fairseq Dense model. The Fairseq Dense model however lacks the broader data that OPT does have. The biggest downfall of OPT is its license, which prohibits any commercial usage, or usage beyond research purposes. |\n", "| [Neo(X)](https://huggingface.co/EleutherAI/gpt-neox-20b) by EleutherAI | Generic | NeoX is the largest EleutherAI model currently available, being a generic model it is not particularly trained towards anything and can do a variety of writing, Q&A and coding tasks. 20B's performance is closely compared to the 13B models and it is worth trying both especially if you have a task that does not involve english writing. Its behavior will be similar to the GPT-J-6B model since they are trained on the same dataset but with more sensitivity towards repetition penalty and with more knowledge. |\n", From 71e5d23a5b3619fa3cb02491fe27d80356575b22 Mon Sep 17 00:00:00 2001 From: YellowRoseCx <80486540+YellowRoseCx@users.noreply.github.com> Date: Wed, 5 Apr 2023 21:23:24 -0500 Subject: [PATCH 11/35] Add IP whitelisting to --host --- aiserver.py | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 3 deletions(-) diff --git a/aiserver.py b/aiserver.py index 7cbbc8aca..c22be16c4 100644 --- a/aiserver.py +++ b/aiserver.py @@ -73,6 +73,8 @@ from transformers import StoppingCriteria, GPT2Tokenizer, GPT2LMHeadModel, GPTNeoForCausalLM, GPTNeoModel, AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer, PreTrainedModel, modeling_utils, AutoModelForTokenClassification from transformers import __version__ as transformers_version import transformers +import ipaddress +from functools import wraps try: from transformers.models.opt.modeling_opt import OPTDecoder except: @@ -86,6 +88,9 @@ from io import BytesIO global tpu_mtj_backend +global allowed_ips +allowed_ips = set() # empty set +enable_whitelist = False if lupa.LUA_VERSION[:2] != (5, 4): @@ -1469,13 +1474,15 @@ def spRequest(filename): #==================================================================# def general_startup(override_args=None): global args + global enable_whitelist + global allowed_ips # Parsing Parameters parser = argparse.ArgumentParser(description="KoboldAI Server") parser.add_argument("--remote", action='store_true', help="Optimizes KoboldAI for Remote Play") parser.add_argument("--noaimenu", action='store_true', help="Disables the ability to select the AI") parser.add_argument("--ngrok", action='store_true', help="Optimizes KoboldAI for Remote Play using Ngrok") parser.add_argument("--localtunnel", action='store_true', help="Optimizes KoboldAI for Remote Play using Localtunnel") - parser.add_argument("--host", action='store_true', help="Optimizes KoboldAI for Remote Play without using a proxy service") + parser.add_argument("--host", type=str, default="", nargs="?", const="", help="Optimizes KoboldAI for LAN Remote Play without using a proxy service. --host opens to all LAN. Enable IP whitelisting by using a comma separated IP list. Supports individual IPs, ranges, and subnets --host 127.0.0.1,127.0.0.2,127.0.0.3,192.168.1.0-192.168.1.255,10.0.0.0/24,etc") parser.add_argument("--port", type=int, help="Specify the port on which the application will be joinable") parser.add_argument("--aria2_port", type=int, help="Specify the port on which aria2's RPC interface will be open if aria2 is installed (defaults to 6799)") parser.add_argument("--model", help="Specify the Model Type to skip the Menu") @@ -1525,6 +1532,10 @@ def general_startup(override_args=None): utils.args = args + + + + #load system and user settings for setting in ['user_settings', 'system_settings']: @@ -1602,9 +1613,32 @@ def new_emit(*args, **kwargs): if args.localtunnel: koboldai_vars.host = True; + if args.host == "": + koboldai_vars.host = True + args.unblock = True if args.host: - koboldai_vars.host = True; - args.unblock = True; + # This means --host option was submitted without an argument + # Enable all LAN IPs (0.0.0.0/0) + if args.host != "": + # Check if --host option was submitted with an argument + # Parse the supplied IP(s) and add them to the allowed IPs list + koboldai_vars.host = True + args.unblock = True + enable_whitelist = True + for ip_str in args.host.split(","): + if "/" in ip_str: + allowed_ips |= set(str(ip) for ip in ipaddress.IPv4Network(ip_str, strict=False).hosts()) + elif "-" in ip_str: + start_ip, end_ip = ip_str.split("-") + start_ip_int = int(ipaddress.IPv4Address(start_ip)) + end_ip_int = int(ipaddress.IPv4Address(end_ip)) + allowed_ips |= set(str(ipaddress.IPv4Address(ip)) for ip in range(start_ip_int, end_ip_int + 1)) + else: + allowed_ips.add(ip_str.strip()) + # Sort and print the allowed IPs list + allowed_ips = sorted(allowed_ips, key=lambda ip: int(''.join([i.zfill(3) for i in ip.split('.')]))) + print(f"Allowed IPs: {allowed_ips}") + if args.cpu: koboldai_vars.use_colab_tpu = False @@ -3450,22 +3484,52 @@ def tpumtjgenerate_settings_callback() -> dict: print(format(colors.GREEN) + "KoboldAI has finished loading and is available at the following link for UI 1: " + koboldai_vars.cloudflare_link + format(colors.END)) print(format(colors.GREEN) + "KoboldAI has finished loading and is available at the following link for UI 2: " + koboldai_vars.cloudflare_link + "/new_ui" + format(colors.END)) + +# Setup IP Whitelisting +# Define a function to check if IP is allowed +def is_allowed_ip(): + global allowed_ips + client_ip = request.remote_addr + if request.path != '/genre_data.json': + print("Connection Attempt: " + request.remote_addr) + print("Allowed?: ", request.remote_addr in allowed_ips) + return client_ip in allowed_ips + + + +# Define a decorator to enforce IP whitelisting +def require_allowed_ip(func): + @wraps(func) + def decorated(*args, **kwargs): + + if enable_whitelist and not is_allowed_ip(): + return abort(403) + return func(*args, **kwargs) + return decorated + + + + # Set up Flask routes @app.route('/') @app.route('/index') +@require_allowed_ip def index(): if args.no_ui: return redirect('/api/latest') else: return render_template('index.html', hide_ai_menu=args.noaimenu) @app.route('/api', strict_slashes=False) +@require_allowed_ip def api(): return redirect('/api/latest') @app.route('/favicon.ico') + def favicon(): return send_from_directory(app.root_path, 'koboldai.ico', mimetype='image/vnd.microsoft.icon') @app.route('/download') +@require_allowed_ip def download(): if args.no_ui: raise NotFound() @@ -4126,6 +4190,8 @@ def execute_outmod(): #==================================================================# @socketio.on('connect') def do_connect(): + print("Connection Attempt: " + request.remote_addr) + print("Allowed?: ", request.remote_addr in allowed_ips) if request.args.get("rely") == "true": return logger.info("Client connected! UI_{}".format(request.args.get('ui'))) @@ -8021,6 +8087,7 @@ def show_folder_usersripts(data): # UI V2 CODE #==================================================================# @app.route('/new_ui') +@require_allowed_ip @logger.catch def new_ui_index(): if args.no_ui: @@ -8048,6 +8115,7 @@ def ui2_connect(): # UI V2 CODE Themes #==================================================================# @app.route('/themes/') +#@require_allowed_ip @logger.catch def ui2_serve_themes(path): return send_from_directory('themes', path) @@ -8086,6 +8154,7 @@ def upload_file(data): get_files_folders(session['current_folder']) @app.route("/upload_kai_story/", methods=["POST"]) +@require_allowed_ip @logger.catch def UI_2_upload_kai_story(file_name: str): @@ -8549,6 +8618,7 @@ def directory_to_zip_data(directory: str, overrides: Optional[dict]) -> bytes: # Save story to json #==================================================================# @app.route("/story_download") +@require_allowed_ip @logger.catch def UI_2_download_story(): if args.no_ui: @@ -9011,6 +9081,7 @@ def UI_2_delete_wi_folder(folder): # Event triggered when user exports world info folder #==================================================================# @app.route('/export_world_info_folder') +@require_allowed_ip @logger.catch def UI_2_export_world_info_folder(): if 'folder' in request.args: @@ -9038,6 +9109,7 @@ def UI_2_upload_world_info_folder(data): koboldai_vars.calc_ai_text() @app.route("/upload_wi", methods=["POST"]) +@require_allowed_ip @logger.catch def UI_2_import_world_info(): wi_data = request.get_json() @@ -9115,6 +9187,7 @@ def UI_2_update_wi_keys(data): socketio.emit("world_info_entry", koboldai_vars.worldinfo_v2.world_info[uid], broadcast=True, room="UI_2") @app.route("/set_wi_image/", methods=["POST"]) +@require_allowed_ip @logger.catch def UI_2_set_wi_image(uid): if uid < 0: @@ -9146,6 +9219,7 @@ def UI_2_set_wi_image(uid): return ":)" @app.route("/get_wi_image/", methods=["GET"]) +@require_allowed_ip @logger.catch def UI_2_get_wi_image(uid): if args.no_ui: @@ -9157,6 +9231,7 @@ def UI_2_get_wi_image(uid): return ":( Couldn't find image", 204 @app.route("/set_commentator_picture/", methods=["POST"]) +@require_allowed_ip @logger.catch def UI_2_set_commentator_image(commentator_id): data = request.get_data() @@ -9165,6 +9240,7 @@ def UI_2_set_commentator_image(commentator_id): return ":)" @app.route("/image_db.json", methods=["GET"]) +@require_allowed_ip @logger.catch def UI_2_get_image_db(): if args.no_ui: @@ -9175,6 +9251,7 @@ def UI_2_get_image_db(): return jsonify([]) @app.route("/action_composition.json", methods=["GET"]) +@require_allowed_ip @logger.catch def UI_2_get_action_composition(): if args.no_ui: @@ -9200,6 +9277,7 @@ def UI_2_get_action_composition(): return jsonify(ret) @app.route("/generated_images/") +@require_allowed_ip def UI_2_send_generated_images(path): return send_from_directory(koboldai_vars.save_paths.generated_images, path) @@ -9509,6 +9587,7 @@ def UI_2_generate_wi(data): socketio.emit("generated_wi", {"uid": uid, "field": field, "out": out_text}, room="UI_2") @app.route("/generate_raw", methods=["GET"]) +@require_allowed_ip def UI_2_generate_raw(): prompt = request.args.get("prompt") @@ -10210,6 +10289,7 @@ def UI_2_privacy_mode(data): # Genres #==================================================================# @app.route("/genre_data.json", methods=["GET"]) +@require_allowed_ip def UI_2_get_applicable_genres(): with open("data/genres.json", "r") as file: genre_list = json.load(file) @@ -10275,12 +10355,14 @@ def UI_2_get_log(data): emit("log_message", web_log_history) @app.route("/get_log") +@require_allowed_ip def UI_2_get_log_get(): if args.no_ui: return redirect('/api/latest') return {'aiserver_log': web_log_history} @app.route("/test_match") +@require_allowed_ip @logger.catch def UI_2_test_match(): koboldai_vars.assign_world_info_to_actions() @@ -10290,6 +10372,7 @@ def UI_2_test_match(): # Download of the audio file #==================================================================# @app.route("/audio") +@require_allowed_ip @logger.catch def UI_2_audio(): if args.no_ui: @@ -10316,6 +10399,7 @@ def UI_2_audio(): # Download of the image for an action #==================================================================# @app.route("/action_image") +@require_allowed_ip @logger.catch def UI_2_action_image(): if args.no_ui: @@ -10375,6 +10459,7 @@ def model_info(): return {"Model Type": "Read Only", "Model Size": "0", "Model Name": koboldai_vars.model.replace("_", "/")} @app.route("/vars") +@require_allowed_ip @logger.catch def show_vars(): if args.no_ui: From ac98cd6dd1b9f0d3372694793e11716501521edb Mon Sep 17 00:00:00 2001 From: YellowRoseCx <80486540+YellowRoseCx@users.noreply.github.com> Date: Wed, 5 Apr 2023 21:27:59 -0500 Subject: [PATCH 12/35] add IP_whitelisting to koboldai_settings.py --- koboldai_settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index d1bc1e5db..be86e3021 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -19,6 +19,8 @@ serverstarted = False queue = None multi_story = False +global enable_whitelist +enable_whitelist = False if importlib.util.find_spec("tortoise") is not None: from tortoise import api @@ -654,6 +656,7 @@ class model_settings(settings): default_settings = {"rep_pen" : 1.1, "rep_pen_slope": 0.7, "rep_pen_range": 1024, "temp": 0.5, "top_p": 0.9, "top_k": 0, "top_a": 0.0, "tfs": 1.0, "typical": 1.0, "sampler_order": [6,0,1,2,3,4,5]} def __init__(self, socketio, koboldai_vars): + self.enable_whitelist = False self._socketio = socketio self.reset_for_model_load() self.model = "" # Model ID string chosen at startup @@ -2745,4 +2748,4 @@ def wi_images(self) -> str: ] } badwordsids_default = [[6880], [50256], [42496], [4613], [17414], [22039], [16410], [27], [29], [38430], [37922], [15913], [24618], [28725], [58], [47175], [36937], [26700], [12878], [16471], [37981], [5218], [29795], [13412], [45160], [3693], [49778], [4211], [20598], [36475], [33409], [44167], [32406], [29847], [29342], [42669], [685], [25787], [7359], [3784], [5320], [33994], [33490], [34516], [43734], [17635], [24293], [9959], [23785], [21737], [28401], [18161], [26358], [32509], [1279], [38155], [18189], [26894], [6927], [14610], [23834], [11037], [14631], [26933], [46904], [22330], [25915], [47934], [38214], [1875], [14692], [41832], [13163], [25970], [29565], [44926], [19841], [37250], [49029], [9609], [44438], [16791], [17816], [30109], [41888], [47527], [42924], [23984], [49074], [33717], [31161], [49082], [30138], [31175], [12240], [14804], [7131], [26076], [33250], [3556], [38381], [36338], [32756], [46581], [17912], [49146]] # Tokenized array of badwords used to prevent AI artifacting -badwordsids_neox = [[0], [1], [44162], [9502], [12520], [31841], [36320], [49824], [34417], [6038], [34494], [24815], [26635], [24345], [3455], [28905], [44270], [17278], [32666], [46880], [7086], [43189], [37322], [17778], [20879], [49821], [3138], [14490], [4681], [21391], [26786], [43134], [9336], [683], [48074], [41256], [19181], [29650], [28532], [36487], [45114], [46275], [16445], [15104], [11337], [1168], [5647], [29], [27482], [44965], [43782], [31011], [42944], [47389], [6334], [17548], [38329], [32044], [35487], [2239], [34761], [7444], [1084], [12399], [18990], [17636], [39083], [1184], [35830], [28365], [16731], [43467], [47744], [1138], [16079], [40116], [45564], [18297], [42368], [5456], [18022], [42696], [34476], [23505], [23741], [39334], [37944], [45382], [38709], [33440], [26077], [43600], [34418], [36033], [6660], [48167], [48471], [15775], [19884], [41533], [1008], [31053], [36692], [46576], [20095], [20629], [31759], [46410], [41000], [13488], [30952], [39258], [16160], [27655], [22367], [42767], [43736], [49694], [13811], [12004], [46768], [6257], [37471], [5264], [44153], [33805], [20977], [21083], [25416], [14277], [31096], [42041], [18331], [33376], [22372], [46294], [28379], [38475], [1656], [5204], [27075], [50001], [16616], [11396], [7748], [48744], [35402], [28120], [41512], [4207], [43144], [14767], [15640], [16595], [41305], [44479], [38958], [18474], [22734], [30522], [46267], [60], [13976], [31830], [48701], [39822], [9014], [21966], [31422], [28052], [34607], [2479], [3851], [32214], [44082], [45507], [3001], [34368], [34758], [13380], [38363], [4299], [46802], [30996], [12630], [49236], [7082], [8795], [5218], [44740], [9686], [9983], [45301], [27114], [40125], [1570], [26997], [544], [5290], [49193], [23781], [14193], [40000], [2947], [43781], [9102], [48064], [42274], [18772], [49384], [9884], [45635], [43521], [31258], [32056], [47686], [21760], [13143], [10148], [26119], [44308], [31379], [36399], [23983], [46694], [36134], [8562], [12977], [35117], [28591], [49021], [47093], [28653], [29013], [46468], [8605], [7254], [25896], [5032], [8168], [36893], [38270], [20499], [27501], [34419], [29547], [28571], [36586], [20871], [30537], [26842], [21375], [31148], [27618], [33094], [3291], [31789], [28391], [870], [9793], [41361], [47916], [27468], [43856], [8850], [35237], [15707], [47552], [2730], [41449], [45488], [3073], [49806], [21938], [24430], [22747], [20924], [46145], [20481], [20197], [8239], [28231], [17987], [42804], [47269], [29972], [49884], [21382], [46295], [36676], [34616], [3921], [26991], [27720], [46265], [654], [9855], [40354], [5291], [34904], [44342], [2470], [14598], [880], [19282], [2498], [24237], [21431], [16369], [8994], [44524], [45662], [13663], [37077], [1447], [37786], [30863], [42854], [1019], [20322], [4398], [12159], [44072], [48664], [31547], [18736], [9259], [31], [16354], [21810], [4357], [37982], [5064], [2033], [32871], [47446], [62], [22158], [37387], [8743], [47007], [17981], [11049], [4622], [37916], [36786], [35138], [29925], [14157], [18095], [27829], [1181], [22226], [5709], [4725], [30189], [37014], [1254], [11380], [42989], [696], [24576], [39487], [30119], [1092], [8088], [2194], [9899], [14412], [21828], [3725], [13544], [5180], [44679], [34398], [3891], [28739], [14219], [37594], [49550], [11326], [6904], [17266], [5749], [10174], [23405], [9955], [38271], [41018], [13011], [48392], [36784], [24254], [21687], [23734], [5413], [41447], [45472], [10122], [17555], [15830], [47384], [12084], [31350], [47940], [11661], [27988], [45443], [905], [49651], [16614], [34993], [6781], [30803], [35869], [8001], [41604], [28118], [46462], [46762], [16262], [17281], [5774], [10943], [5013], [18257], [6750], [4713], [3951], [11899], [38791], [16943], [37596], [9318], [18413], [40473], [13208], [16375]] \ No newline at end of file +badwordsids_neox = [[0], [1], [44162], [9502], [12520], [31841], [36320], [49824], [34417], [6038], [34494], [24815], [26635], [24345], [3455], [28905], [44270], [17278], [32666], [46880], [7086], [43189], [37322], [17778], [20879], [49821], [3138], [14490], [4681], [21391], [26786], [43134], [9336], [683], [48074], [41256], [19181], [29650], [28532], [36487], [45114], [46275], [16445], [15104], [11337], [1168], [5647], [29], [27482], [44965], [43782], [31011], [42944], [47389], [6334], [17548], [38329], [32044], [35487], [2239], [34761], [7444], [1084], [12399], [18990], [17636], [39083], [1184], [35830], [28365], [16731], [43467], [47744], [1138], [16079], [40116], [45564], [18297], [42368], [5456], [18022], [42696], [34476], [23505], [23741], [39334], [37944], [45382], [38709], [33440], [26077], [43600], [34418], [36033], [6660], [48167], [48471], [15775], [19884], [41533], [1008], [31053], [36692], [46576], [20095], [20629], [31759], [46410], [41000], [13488], [30952], [39258], [16160], [27655], [22367], [42767], [43736], [49694], [13811], [12004], [46768], [6257], [37471], [5264], [44153], [33805], [20977], [21083], [25416], [14277], [31096], [42041], [18331], [33376], [22372], [46294], [28379], [38475], [1656], [5204], [27075], [50001], [16616], [11396], [7748], [48744], [35402], [28120], [41512], [4207], [43144], [14767], [15640], [16595], [41305], [44479], [38958], [18474], [22734], [30522], [46267], [60], [13976], [31830], [48701], [39822], [9014], [21966], [31422], [28052], [34607], [2479], [3851], [32214], [44082], [45507], [3001], [34368], [34758], [13380], [38363], [4299], [46802], [30996], [12630], [49236], [7082], [8795], [5218], [44740], [9686], [9983], [45301], [27114], [40125], [1570], [26997], [544], [5290], [49193], [23781], [14193], [40000], [2947], [43781], [9102], [48064], [42274], [18772], [49384], [9884], [45635], [43521], [31258], [32056], [47686], [21760], [13143], [10148], [26119], [44308], [31379], [36399], [23983], [46694], [36134], [8562], [12977], [35117], [28591], [49021], [47093], [28653], [29013], [46468], [8605], [7254], [25896], [5032], [8168], [36893], [38270], [20499], [27501], [34419], [29547], [28571], [36586], [20871], [30537], [26842], [21375], [31148], [27618], [33094], [3291], [31789], [28391], [870], [9793], [41361], [47916], [27468], [43856], [8850], [35237], [15707], [47552], [2730], [41449], [45488], [3073], [49806], [21938], [24430], [22747], [20924], [46145], [20481], [20197], [8239], [28231], [17987], [42804], [47269], [29972], [49884], [21382], [46295], [36676], [34616], [3921], [26991], [27720], [46265], [654], [9855], [40354], [5291], [34904], [44342], [2470], [14598], [880], [19282], [2498], [24237], [21431], [16369], [8994], [44524], [45662], [13663], [37077], [1447], [37786], [30863], [42854], [1019], [20322], [4398], [12159], [44072], [48664], [31547], [18736], [9259], [31], [16354], [21810], [4357], [37982], [5064], [2033], [32871], [47446], [62], [22158], [37387], [8743], [47007], [17981], [11049], [4622], [37916], [36786], [35138], [29925], [14157], [18095], [27829], [1181], [22226], [5709], [4725], [30189], [37014], [1254], [11380], [42989], [696], [24576], [39487], [30119], [1092], [8088], [2194], [9899], [14412], [21828], [3725], [13544], [5180], [44679], [34398], [3891], [28739], [14219], [37594], [49550], [11326], [6904], [17266], [5749], [10174], [23405], [9955], [38271], [41018], [13011], [48392], [36784], [24254], [21687], [23734], [5413], [41447], [45472], [10122], [17555], [15830], [47384], [12084], [31350], [47940], [11661], [27988], [45443], [905], [49651], [16614], [34993], [6781], [30803], [35869], [8001], [41604], [28118], [46462], [46762], [16262], [17281], [5774], [10943], [5013], [18257], [6750], [4713], [3951], [11899], [38791], [16943], [37596], [9318], [18413], [40473], [13208], [16375]] From bde9c6980fadd5670a045fe7741649dcc116c993 Mon Sep 17 00:00:00 2001 From: Henk Date: Fri, 14 Apr 2023 14:13:46 +0200 Subject: [PATCH 13/35] Transformers 4.28 support --- aiserver.py | 2 +- environments/huggingface.yml | 2 +- environments/rocm.yml | 2 +- requirements.txt | 2 +- requirements_mtj.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aiserver.py b/aiserver.py index c22be16c4..2ec6d8172 100644 --- a/aiserver.py +++ b/aiserver.py @@ -2951,7 +2951,7 @@ def get_original_key(key): koboldai_vars.status_message = "Loading model" koboldai_vars.total_layers = num_tensors koboldai_vars.loaded_layers = 0 - utils.bar = tqdm(total=num_tensors, desc="Loading model tensors", file=Send_to_socketio()) + utils.bar = tqdm(total=num_tensors, desc="Loading model tensors", file=Send_to_socketio(), position=1) with zipfile.ZipFile(f, "r") as z: try: diff --git a/environments/huggingface.yml b/environments/huggingface.yml index 2eb1bfe0c..138993dde 100644 --- a/environments/huggingface.yml +++ b/environments/huggingface.yml @@ -30,7 +30,7 @@ dependencies: - flask-ngrok - flask-cors - lupa==1.10 - - transformers==4.25.1 + - transformers==4.28.0 - huggingface_hub==0.12.1 - safetensors - accelerate==0.18.0 diff --git a/environments/rocm.yml b/environments/rocm.yml index 43fd331f6..e28c86cba 100644 --- a/environments/rocm.yml +++ b/environments/rocm.yml @@ -29,7 +29,7 @@ dependencies: - flask-ngrok - flask-cors - lupa==1.10 - - transformers==4.25.1 + - transformers==4.28.0 - huggingface_hub==0.12.1 - safetensors - accelerate diff --git a/requirements.txt b/requirements.txt index d85c61fda..c2a61ca6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -transformers==4.25.1 +transformers==4.28.0 huggingface_hub==0.12.1 Flask==2.2.3 Flask-SocketIO==5.3.2 diff --git a/requirements_mtj.txt b/requirements_mtj.txt index 076d94290..37b76a231 100644 --- a/requirements_mtj.txt +++ b/requirements_mtj.txt @@ -5,7 +5,7 @@ requests dm-haiku == 0.0.5 jax == 0.2.21 jaxlib >= 0.1.69, <= 0.3.7 -transformers == 4.25.1 +transformers == 4.28.0 chex == 0.1.5 huggingface_hub==0.12.1 progressbar2 From 3eda7269f72bfa954a13aebb9d965b9c7dad9e61 Mon Sep 17 00:00:00 2001 From: Henk Date: Sat, 15 Apr 2023 14:58:24 +0200 Subject: [PATCH 14/35] Fix incorrect host merge --- aiserver.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/aiserver.py b/aiserver.py index 2ec6d8172..59cfac0ce 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1613,9 +1613,6 @@ def new_emit(*args, **kwargs): if args.localtunnel: koboldai_vars.host = True; - if args.host == "": - koboldai_vars.host = True - args.unblock = True if args.host: # This means --host option was submitted without an argument # Enable all LAN IPs (0.0.0.0/0) From 67334bd69848bc8f3c00f1015f9f95170d2c98a3 Mon Sep 17 00:00:00 2001 From: Henk Date: Sat, 15 Apr 2023 17:45:00 +0200 Subject: [PATCH 15/35] Pin accelerate version --- environments/rocm.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environments/rocm.yml b/environments/rocm.yml index e28c86cba..c3e95496f 100644 --- a/environments/rocm.yml +++ b/environments/rocm.yml @@ -32,7 +32,7 @@ dependencies: - transformers==4.28.0 - huggingface_hub==0.12.1 - safetensors - - accelerate + - accelerate==0.18.0 - git+https://github.com/VE-FORBRYDERNE/mkultra - ansi2html - flask_compress diff --git a/requirements.txt b/requirements.txt index c2a61ca6a..234686563 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ markdown bleach==4.1.0 sentencepiece protobuf -accelerate +accelerate==0.18.0 flask-session==0.4.0 marshmallow>=3.13 apispec-webframeworks From b68860b3de1adef4e162834bd524c92e39dbe264 Mon Sep 17 00:00:00 2001 From: Henk Date: Sat, 15 Apr 2023 18:31:39 +0200 Subject: [PATCH 16/35] Workaround to make --host work again --- aiserver.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/aiserver.py b/aiserver.py index 59cfac0ce..886a802e5 100644 --- a/aiserver.py +++ b/aiserver.py @@ -1482,7 +1482,7 @@ def general_startup(override_args=None): parser.add_argument("--noaimenu", action='store_true', help="Disables the ability to select the AI") parser.add_argument("--ngrok", action='store_true', help="Optimizes KoboldAI for Remote Play using Ngrok") parser.add_argument("--localtunnel", action='store_true', help="Optimizes KoboldAI for Remote Play using Localtunnel") - parser.add_argument("--host", type=str, default="", nargs="?", const="", help="Optimizes KoboldAI for LAN Remote Play without using a proxy service. --host opens to all LAN. Enable IP whitelisting by using a comma separated IP list. Supports individual IPs, ranges, and subnets --host 127.0.0.1,127.0.0.2,127.0.0.3,192.168.1.0-192.168.1.255,10.0.0.0/24,etc") + parser.add_argument("--host", type=str, default="Disabled", nargs="?", const="", help="Optimizes KoboldAI for LAN Remote Play without using a proxy service. --host opens to all LAN. Enable IP whitelisting by using a comma separated IP list. Supports individual IPs, ranges, and subnets --host 127.0.0.1,127.0.0.2,127.0.0.3,192.168.1.0-192.168.1.255,10.0.0.0/24,etc") parser.add_argument("--port", type=int, help="Specify the port on which the application will be joinable") parser.add_argument("--aria2_port", type=int, help="Specify the port on which aria2's RPC interface will be open if aria2 is installed (defaults to 6799)") parser.add_argument("--model", help="Specify the Model Type to skip the Menu") @@ -1613,14 +1613,14 @@ def new_emit(*args, **kwargs): if args.localtunnel: koboldai_vars.host = True; - if args.host: + if args.host != "Disabled": # This means --host option was submitted without an argument # Enable all LAN IPs (0.0.0.0/0) + koboldai_vars.host = True + args.unblock = True if args.host != "": # Check if --host option was submitted with an argument # Parse the supplied IP(s) and add them to the allowed IPs list - koboldai_vars.host = True - args.unblock = True enable_whitelist = True for ip_str in args.host.split(","): if "/" in ip_str: @@ -1637,6 +1637,7 @@ def new_emit(*args, **kwargs): print(f"Allowed IPs: {allowed_ips}") + if args.cpu: koboldai_vars.use_colab_tpu = False From 860b697a705ad28982b20aac1135e4ff35e50647 Mon Sep 17 00:00:00 2001 From: Syler Clayton <2224238+Relys@users.noreply.github.com> Date: Sat, 15 Apr 2023 09:51:45 -0700 Subject: [PATCH 17/35] Update install_requirements.sh Made parameter case insensitive. --- install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_requirements.sh b/install_requirements.sh index 7d2c0c2fd..e131ad37e 100755 --- a/install_requirements.sh +++ b/install_requirements.sh @@ -1,12 +1,12 @@ #!/bin/bash -if [[ $1 = "cuda" ]]; then +if [[ $1 = "cuda" ]||[ $1 = "CUDA" ]]; then wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba bin/micromamba create -f environments/huggingface.yml -r runtime -n koboldai -y # Weird micromamba bug causes it to fail the first time, running it twice just to be safe, the second time is much faster bin/micromamba create -f environments/huggingface.yml -r runtime -n koboldai -y exit fi -if [[ $1 = "rocm" ]]; then +if [[ $1 = "rocm" ]||[ $1 = "ROCM" ]]; then wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba bin/micromamba create -f environments/rocm.yml -r runtime -n koboldai-rocm -y # Weird micromamba bug causes it to fail the first time, running it twice just to be safe, the second time is much faster From b1d3a37fb082197115765cc16e8faa074f1920b4 Mon Sep 17 00:00:00 2001 From: Henk Date: Sun, 16 Apr 2023 19:31:17 +0200 Subject: [PATCH 18/35] <|comments|> now are [<|comments|>] to support OA --- koboldai_settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index be86e3021..fdd89ba91 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -1204,13 +1204,13 @@ def __setattr__(self, name, value): class system_settings(settings): local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', - 'comregex_ui', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', + 'sp', '_horde_pid', 'inference_config', 'image_pipeline', 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states'] no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', 'summary_tokenizer', 'use_colab_tpu', 'noai', 'disable_set_aibusy', 'cloudflare_link', 'tts_model', - 'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu', 'rng_states'] + 'generating_image', 'bit_8_available', 'host', 'hascuda', 'usegpu', 'rng_states', 'comregex_ai', 'comregex_ui'] settings_name = "system" def __init__(self, socketio, koboldai_var): self._socketio = socketio @@ -1252,8 +1252,8 @@ def __init__(self, socketio, koboldai_var): self.regex_sl = re.compile(r'\n*(?<=.) *\n(.|\n)*') # Pattern for limiting the output to a single line self.acregex_ai = re.compile(r'\n* *>(.|\n)*') # Pattern for matching adventure actions from the AI so we can remove them self.acregex_ui = re.compile(r'^ *(>.*)$', re.MULTILINE) # Pattern for matching actions in the HTML-escaped story so we can apply colouring, etc (make sure to encase part to format in parentheses) - self.comregex_ai = re.compile(r'(?:\n<\|(?:.|\n)*?\|>(?=\n|$))|(?:<\|(?:.|\n)*?\|>\n?)') # Pattern for matching comments to remove them before sending them to the AI - self.comregex_ui = re.compile(r'(<\|(?:.|\n)*?\|>)') # Pattern for matching comments in the editor + self.comregex_ai = re.compile(r'(?:\n\[<\|(?:.|\n)*?\|>\](?=\n|$))|(?:\[<\|(?:.|\n)*?\|>\]\n?)') # Pattern for matching comments to remove them before sending them to the AI + self.comregex_ui = re.compile(r'(\[<\|(?:.|\n)*?\|>\])') # Pattern for matching comments in the editor self.host = False self.flaskwebgui = False self.quiet = False # If set will suppress any story text from being printed to the console (will only be seen on the client web page) From 2926dac72b4c8cd5a2c93eaad4d901cc109e66d4 Mon Sep 17 00:00:00 2001 From: Henk Date: Sun, 16 Apr 2023 23:59:13 +0200 Subject: [PATCH 19/35] Don't print allowed_ips if unused --- aiserver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aiserver.py b/aiserver.py index 886a802e5..b8ff4f3e5 100644 --- a/aiserver.py +++ b/aiserver.py @@ -3490,7 +3490,8 @@ def is_allowed_ip(): client_ip = request.remote_addr if request.path != '/genre_data.json': print("Connection Attempt: " + request.remote_addr) - print("Allowed?: ", request.remote_addr in allowed_ips) + if allowed_ips: + print("Allowed?: ", request.remote_addr in allowed_ips) return client_ip in allowed_ips @@ -4189,7 +4190,8 @@ def execute_outmod(): @socketio.on('connect') def do_connect(): print("Connection Attempt: " + request.remote_addr) - print("Allowed?: ", request.remote_addr in allowed_ips) + if allowed_ips: + print("Allowed?: ", request.remote_addr in allowed_ips) if request.args.get("rely") == "true": return logger.info("Client connected! UI_{}".format(request.args.get('ui'))) From fcba26f631e7efcd4a92de206d2bd6da40ed65c3 Mon Sep 17 00:00:00 2001 From: ebolam Date: Sun, 16 Apr 2023 20:32:51 -0400 Subject: [PATCH 20/35] Fix for uploading files in UI2 --- koboldai_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index fdd89ba91..06a15e4e8 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -1205,7 +1205,7 @@ class system_settings(settings): local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', - 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states'] + 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui'] no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', From a34ce85d21ac750b6a0b69a3e33ff24e9179a224 Mon Sep 17 00:00:00 2001 From: ebolam Date: Sun, 16 Apr 2023 20:43:03 -0400 Subject: [PATCH 21/35] Fix for UI1 remote mode file loading from browser --- aiserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aiserver.py b/aiserver.py index b8ff4f3e5..9dd621ef3 100644 --- a/aiserver.py +++ b/aiserver.py @@ -7454,13 +7454,13 @@ def loadRequest(loadpath, filename=None): if not loadpath: return - + #Original UI only sends the story name and assumes it's always a .json file... here we check to see if it's a directory to load that way - if not os.path.exists(loadpath): + if not isinstance(loadpath, dict) and not os.path.exists(loadpath): if os.path.exists(loadpath.replace(".json", "")): loadpath = loadpath.replace(".json", "") - if os.path.isdir(loadpath): + if not isinstance(loadpath, dict) and os.path.isdir(loadpath): if not valid_v3_story(loadpath): raise RuntimeError(f"Tried to load {loadpath}, a non-save directory.") koboldai_vars.update_story_path_structure(loadpath) From d16a3f4dc3746e65bc16e92d55b91dc15d7b3641 Mon Sep 17 00:00:00 2001 From: ebolam Date: Sun, 16 Apr 2023 21:02:42 -0400 Subject: [PATCH 22/35] Added comregex_ui and comregex_ai to the UI under other settings. Needs proper titles and descriptions still. --- gensettings.py | 32 ++++++++++++++++++++++++++++++++ koboldai_settings.py | 16 ++++++++++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/gensettings.py b/gensettings.py index 8d68b4b53..a0a20908e 100644 --- a/gensettings.py +++ b/gensettings.py @@ -881,6 +881,38 @@ "classname": "system", "name": "seed", "extra_classes": "var_sync_alt_system_seed_specified", + "ui_level": 2 + }, + { + "uitype": "text", + "unit": "text", + "label": "comregex_ai_string", + "id": "comregex_ai_string", + "min": 0, + "max": 1, + "step": 1, + "default": 1, + "tooltip": "Pattern for matching comments to remove them before sending them to the AI.", + "menu_path": "Settings", + "sub_path": "Other", + "classname": "system", + "name": "comregex_ai_string", + "ui_level": 2 + }, + { + "uitype": "text", + "unit": "text", + "label": "comregex_ui_string", + "id": "comregex_ui_string", + "min": 0, + "max": 1, + "step": 1, + "default": 1, + "tooltip": "Pattern for matching comments in the editor.", + "menu_path": "Settings", + "sub_path": "Other", + "classname": "system", + "name": "comregex_ui_string", "ui_level": 2 }, { diff --git a/koboldai_settings.py b/koboldai_settings.py index 06a15e4e8..407ae1e1c 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -1203,9 +1203,9 @@ def __setattr__(self, name, value): class system_settings(settings): local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', - 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', + 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'comregex_ui', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', - 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui'] + 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states'] no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', @@ -1252,8 +1252,10 @@ def __init__(self, socketio, koboldai_var): self.regex_sl = re.compile(r'\n*(?<=.) *\n(.|\n)*') # Pattern for limiting the output to a single line self.acregex_ai = re.compile(r'\n* *>(.|\n)*') # Pattern for matching adventure actions from the AI so we can remove them self.acregex_ui = re.compile(r'^ *(>.*)$', re.MULTILINE) # Pattern for matching actions in the HTML-escaped story so we can apply colouring, etc (make sure to encase part to format in parentheses) - self.comregex_ai = re.compile(r'(?:\n\[<\|(?:.|\n)*?\|>\](?=\n|$))|(?:\[<\|(?:.|\n)*?\|>\]\n?)') # Pattern for matching comments to remove them before sending them to the AI - self.comregex_ui = re.compile(r'(\[<\|(?:.|\n)*?\|>\])') # Pattern for matching comments in the editor + self.comregex_ai_string = '(?:\n\[<\|(?:.|\n)*?\|>\](?=\n|$))|(?:\[<\|(?:.|\n)*?\|>\]\n?)' # Pattern for matching comments to remove them before sending them to the AI + self.comregex_ui_string = '(\[<\|(?:.|\n)*?\|>\])' # Pattern for matching comments in the editor + self.comregex_ai = re.compile(self.comregex_ai_string) # Pattern for matching comments to remove them before sending them to the AI + self.comregex_ui = re.compile(self.comregex_ui_string) # Pattern for matching comments in the editor self.host = False self.flaskwebgui = False self.quiet = False # If set will suppress any story text from being printed to the console (will only be seen on the client web page) @@ -1340,6 +1342,12 @@ def __setattr__(self, name, value): self._socketio.emit('from_server', {'cmd': 'spstatitems', 'data': {self.spfilename: self.spmeta} if self.allowsp and len(self.spfilename) else {}}, namespace=None, broadcast=True, room="UI_1") super().__setattr__("sp_changed", False) + if name == 'comregex_ai_string': + self.comregex_ai = re.compile(self.comregex_ai_string) + + if name == 'comregex_ui_string': + self.comregex_ui = re.compile(self.comregex_ui_string) + if name == 'keep_img_gen_in_memory' and value == False: self.image_pipeline = None From ee6e7e9b726b5c5186fe73430f417e551f5f3bc5 Mon Sep 17 00:00:00 2001 From: henk717 Date: Mon, 17 Apr 2023 22:59:55 +0200 Subject: [PATCH 23/35] Colab description changes --- colab/GPU.ipynb | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/colab/GPU.ipynb b/colab/GPU.ipynb index c6c1e5ba6..3c6b97064 100644 --- a/colab/GPU.ipynb +++ b/colab/GPU.ipynb @@ -39,8 +39,6 @@ "\n", "For more information about KoboldAI check our our Github readme : https://github.com/KoboldAI/KoboldAI-Client/blob/main/readme.md\n", "\n", - "For the larger AI models (That are typically more coherent) check out our **[TPU edition](https://colab.research.google.com/github/KoboldAI/KoboldAI-Client/blob/main/colab/TPU.ipynb)**!\n", - "\n", "---\n", "## How to load KoboldAI: Everything you need to know\n", "1. On a phone? First put your browser in desktop mode because of a Google Colab bug. Otherwise nothing will happen when you click the play button. Then tap the play button next to \"<-- Tap This if you play on Mobile\", you will see an audio player. Keep the audio player playing so Colab does not get shut down in the background.\n", @@ -210,22 +208,6 @@ "| [Fairseq Dense](https://huggingface.co/KoboldAI/fairseq-dense-2.7B) | Generic | Trained by Facebook Researchers this model stems from the MOE research project within Fairseq. This particular version has been converted by us for use in KoboldAI. It is known to be on par with the larger models from EleutherAI and considered as better for pop culture and language tasks. Because the model has never seen a new line (enter) it may perform worse on formatting and paragraphing. Compared to other models the dataset focuses primarily on literature and contains little else. |\n", "| [Neo](https://huggingface.co/EleutherAI/gpt-neo-2.7B) by EleutherAI | Generic | This is the base model for all the other 2.7B models, it is best used when you have a use case that we have no other models available for, such as writing blog articles or programming. It can also be a good basis for the experience of some of the softprompts if your softprompt is not about a subject the other models cover. |\n", "\n", - "# [TPU Edition Model Descriptions](https://colab.research.google.com/github/KoboldAI/KoboldAI-Client/blob/main/colab/TPU.ipynb)\n", - "\n", - "| Model | Style | Description |\n", - "| --- | --- | --- |\n", - "| [Nerys](https://huggingface.co/KoboldAI/fairseq-dense-13B-Nerys) by Mr Seeker | Novel/Adventure | Nerys is a hybrid model based on Pike (A newer Janeway), on top of the Pike dataset you also get some Light Novels, Adventure mode support and a little bit of Shinen thrown in the mix. The end result is a very diverse model that is heavily biased towards SFW novel writing, but one that can go beyond its novel training and make for an excellent adventure model to. Adventure mode is best played from a second person perspective, but can be played in first or third person as well. Novel writing can be done best from the first or third person. |\n", - "| [Erebus](https://huggingface.co/KoboldAI/OPT-13B-Erebus) by Mr Seeker | NSFW | Erebus is our community's flagship NSFW model, being a combination of multiple large datasets that include Literotica, Shinen and erotic novels from Nerys and featuring thourough tagging support it covers the vast majority of erotic writing styles. This model is capable of replacing both the Lit and Shinen models in terms of content and style and has been well received as (one of) the best NSFW models out there. If you wish to use this model for commercial or non research usage we recommend choosing the 20B version as that one is not subject to the restrictive OPT license. |\n", - "| [Janeway](https://huggingface.co/KoboldAI/fairseq-dense-13B-Janeway) by Mr Seeker | Novel | Janeway is a model created from Picard's dataset combined with a brand new collection of ebooks. This model is trained on 20% more content than Picard and has been trained on literature from various genres. Although the model is mainly focussed on SFW, romantic scenes might involve a degree of nudity. |\n", - "| [Shinen](https://huggingface.co/KoboldAI/fairseq-dense-13B-Shinen) by Mr Seeker | NSFW | Shinen is an NSFW model trained on a variety of stories from the website Sexstories it contains many different kinks. It has been merged into the larger (and better) Erebus model. |\n", - "| [Skein](https://huggingface.co/KoboldAI/GPT-J-6B-Skein) by VE\\_FORBRYDERNE | Adventure | Skein is best used with Adventure mode enabled, it consists of a 4 times larger adventure dataset than the Adventure model making it excellent for text adventure gaming. On top of that it also consists of light novel training further expanding its knowledge and writing capabilities. It can be used with the You filter bias if you wish to write Novels with it, but dedicated Novel models can perform better for this task. |\n", - "| [Adventure](https://huggingface.co/KoboldAI/GPT-J-6B-Adventure) by VE\\_FORBRYDERNE | Adventure | Adventure is a 6B model designed to mimick the behavior of AI Dungeon. It is exclusively for Adventure Mode and can take you on the epic and wackey adventures that AI Dungeon players love. It also features the many tropes of AI Dungeon as it has been trained on very similar data. It must be used in second person (You). |\n", - "| [Lit](https://huggingface.co/hakurei/lit-6B) by Haru | NSFW | Lit is a great NSFW model trained by Haru on both a large set of Literotica stories and high quality novels along with tagging support. Creating a high quality model for your NSFW stories. This model is exclusively a novel model and is best used in third person. |\n", - "| [OPT](https://huggingface.co/facebook/opt-13b) by Metaseq | Generic | OPT is considered one of the best base models as far as content goes, its behavior has the strengths of both GPT-Neo and Fairseq Dense. Compared to Neo duplicate and unnecessary content has been left out, while additional literature was added in similar to the Fairseq Dense model. The Fairseq Dense model however lacks the broader data that OPT does have. The biggest downfall of OPT is its license, which prohibits any commercial usage, or usage beyond research purposes. |\n", - "| [Neo(X)](https://huggingface.co/EleutherAI/gpt-neox-20b) by EleutherAI | Generic | NeoX is the largest EleutherAI model currently available, being a generic model it is not particularly trained towards anything and can do a variety of writing, Q&A and coding tasks. 20B's performance is closely compared to the 13B models and it is worth trying both especially if you have a task that does not involve english writing. Its behavior will be similar to the GPT-J-6B model since they are trained on the same dataset but with more sensitivity towards repetition penalty and with more knowledge. |\n", - "| [Fairseq Dense](https://huggingface.co/KoboldAI/fairseq-dense-13B) | Generic | Trained by Facebook Researchers this model stems from the MOE research project within Fairseq. This particular version has been converted by us for use in KoboldAI. It is known to be on par with the larger 20B model from EleutherAI and considered as better for pop culture and language tasks. Because the model has never seen a new line (enter) it may perform worse on formatting and paragraphing. Compared to other models the dataset focuses primarily on literature and contains little else. |\n", - "| [GPT-J-6B](https://huggingface.co/EleutherAI/gpt-j-6B) by EleutherAI | Generic | This model serves as the basis for most other 6B models (Some being based on Fairseq Dense instead). Being trained on the Pile and not biased towards anything in particular it is suitable for a variety of tasks such as writing, Q&A and coding tasks. You will likely get better result with larger generic models or finetuned models. |\n", - "\n", "| Style | Description |\n", "| --------- | ------------------------------------------------------------ |\n", "| Novel | For regular story writing, not compatible with Adventure mode or other specialty modes. |\n", From e4c15fe1f6a2be574eb223e0102cbdb2a11dfe40 Mon Sep 17 00:00:00 2001 From: henk717 Date: Fri, 21 Apr 2023 03:00:52 +0200 Subject: [PATCH 24/35] Update install_requirements.sh --- install_requirements.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_requirements.sh b/install_requirements.sh index e131ad37e..12c835959 100755 --- a/install_requirements.sh +++ b/install_requirements.sh @@ -1,12 +1,12 @@ #!/bin/bash -if [[ $1 = "cuda" ]||[ $1 = "CUDA" ]]; then +if [[ $1 = "cuda" || $1 = "CUDA" ]]; then wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba bin/micromamba create -f environments/huggingface.yml -r runtime -n koboldai -y # Weird micromamba bug causes it to fail the first time, running it twice just to be safe, the second time is much faster bin/micromamba create -f environments/huggingface.yml -r runtime -n koboldai -y exit fi -if [[ $1 = "rocm" ]||[ $1 = "ROCM" ]]; then +if [[ $1 = "rocm" || $1 = "ROCM" ]]; then wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba bin/micromamba create -f environments/rocm.yml -r runtime -n koboldai-rocm -y # Weird micromamba bug causes it to fail the first time, running it twice just to be safe, the second time is much faster From 4502a3f6b8dfe49523a0b94e8f461b5ea6bb208d Mon Sep 17 00:00:00 2001 From: Bogdan Drema Date: Sat, 22 Apr 2023 22:31:21 +0100 Subject: [PATCH 25/35] Fix: TPU driver error to_dlpack/from_dlpack was causing issues with tensor with new jax version --- requirements_mtj.txt | 6 +++--- tpu_mtj_backend.py | 29 ++++++++++++++--------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/requirements_mtj.txt b/requirements_mtj.txt index 37b76a231..19da39109 100644 --- a/requirements_mtj.txt +++ b/requirements_mtj.txt @@ -2,9 +2,9 @@ torch >= 1.9, < 1.13 numpy tqdm requests -dm-haiku == 0.0.5 -jax == 0.2.21 -jaxlib >= 0.1.69, <= 0.3.7 +dm-haiku==0.0.9 +jax==0.3.25 +jaxlib==0.3.25 transformers == 4.28.0 chex == 0.1.5 huggingface_hub==0.12.1 diff --git a/tpu_mtj_backend.py b/tpu_mtj_backend.py index 02754d954..dc0a664d5 100644 --- a/tpu_mtj_backend.py +++ b/tpu_mtj_backend.py @@ -1095,7 +1095,7 @@ def read_neox_checkpoint(state, path, config, checkpoint_shards=2): koboldai_vars.status_message = "" -def load_model(path: str, driver_version="tpu_driver0.1_dev20210607", hf_checkpoint=False, socketio_queue=None, initial_load=False, logger=None, **kwargs) -> None: +def load_model(path: str, driver_version="tpu_driver_nightly", hf_checkpoint=False, socketio_queue=None, initial_load=False, logger=None, **kwargs) -> None: global thread_resources_env, seq, tokenizer, network, params, pad_token_id if "pad_token_id" in kwargs: @@ -1270,11 +1270,6 @@ def check_status(url, queue): logger.message(f"KoboldAI has finished loading and is available at the following link for UI 1: {koboldai_vars.cloudflare_link}") logger.message(f"KoboldAI has finished loading and is available at the following link for UI 2: {koboldai_vars.cloudflare_link}/new_ui") - - global shard_xmap, batch_xmap - shard_xmap = __shard_xmap() - batch_xmap = __batch_xmap(shard_dim=cores_per_replica) - global badwords # These are the tokens that we don't want the AI to ever write badwords = jnp.array(koboldai_vars.badwordsids).squeeze() @@ -1401,19 +1396,20 @@ def callback(model_dict, f, **_): #if "no_transpose" not in transforms and tensor.ndim == 2: # tensor = tensor.T tensor.unsqueeze_(0) - if tensor.dtype is torch.float16 or tensor.dtype is torch.float32: - tensor = tensor.bfloat16() + # Shard the tensor so that parts of the tensor can be used # on different TPU cores + tensor = reshard_reverse( + tensor, + params["cores_per_replica"], + network.state["params"][spec["module"]][spec["param"]].shape, + ) + tensor = jnp.array(tensor.detach()) + if tensor.dtype is torch.float16 or tensor.dtype is torch.float32: + tensor = tensor.bfloat16() network.state["params"][spec["module"]][spec["param"]] = move_xmap( - jax.dlpack.from_dlpack(torch.utils.dlpack.to_dlpack( - reshard_reverse( - tensor, - params["cores_per_replica"], - network.state["params"][spec["module"]][spec["param"]].shape, - ) - )).copy(), + tensor, np.empty(params["cores_per_replica"]), ) @@ -1506,3 +1502,6 @@ def callback(model_dict, f, **_): model = GPTNeoForCausalLM.from_pretrained(koboldai_vars.model, revision=koboldai_vars.revision, cache_dir="cache") #network.state = network.move_xmap(network.state, np.zeros(cores_per_replica)) + global shard_xmap, batch_xmap + shard_xmap = __shard_xmap() + batch_xmap = __batch_xmap(shard_dim=cores_per_replica) \ No newline at end of file From 8f44141f965941fa768a668991d11247a410149a Mon Sep 17 00:00:00 2001 From: henk717 Date: Sun, 23 Apr 2023 01:43:37 +0200 Subject: [PATCH 26/35] Pin driver to the one from JAX 0.3.25 --- tpu_mtj_backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tpu_mtj_backend.py b/tpu_mtj_backend.py index dc0a664d5..d8edb92fa 100644 --- a/tpu_mtj_backend.py +++ b/tpu_mtj_backend.py @@ -1095,7 +1095,7 @@ def read_neox_checkpoint(state, path, config, checkpoint_shards=2): koboldai_vars.status_message = "" -def load_model(path: str, driver_version="tpu_driver_nightly", hf_checkpoint=False, socketio_queue=None, initial_load=False, logger=None, **kwargs) -> None: +def load_model(path: str, driver_version="tpu_driver_20221109", hf_checkpoint=False, socketio_queue=None, initial_load=False, logger=None, **kwargs) -> None: global thread_resources_env, seq, tokenizer, network, params, pad_token_id if "pad_token_id" in kwargs: @@ -1504,4 +1504,4 @@ def callback(model_dict, f, **_): #network.state = network.move_xmap(network.state, np.zeros(cores_per_replica)) global shard_xmap, batch_xmap shard_xmap = __shard_xmap() - batch_xmap = __batch_xmap(shard_dim=cores_per_replica) \ No newline at end of file + batch_xmap = __batch_xmap(shard_dim=cores_per_replica) From 92a0bf9524f831d10b8b1006ee383a300fef5af8 Mon Sep 17 00:00:00 2001 From: Bogdan Drema Date: Sun, 23 Apr 2023 00:49:42 +0100 Subject: [PATCH 27/35] Fix: TPU driver error to_dlpack/from_dlpack was causing issues with tensor with new jax version --- requirements_mtj.txt | 7 ++++--- tpu_mtj_backend.py | 26 ++++++++++++++++---------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/requirements_mtj.txt b/requirements_mtj.txt index 9447541f9..ea399a8f8 100644 --- a/requirements_mtj.txt +++ b/requirements_mtj.txt @@ -2,9 +2,10 @@ torch >= 1.9, < 1.13 numpy tqdm requests -dm-haiku == 0.0.5 -jax == 0.2.21 -jaxlib >= 0.1.69, <= 0.3.7 +dm-haiku==0.0.9 +jax==0.3.25 +jaxlib==0.3.25 +transformers == 4.28.0 chex == 0.1.5 transformers == 4.24.0 huggingface_hub==0.12.1 diff --git a/tpu_mtj_backend.py b/tpu_mtj_backend.py index 4b27493e6..f878d6909 100644 --- a/tpu_mtj_backend.py +++ b/tpu_mtj_backend.py @@ -1049,7 +1049,7 @@ def read_neox_checkpoint(state, path, config, checkpoint_shards=2): raise RuntimeError(error) -def load_model(path: str, driver_version="tpu_driver0.1_dev20210607", hf_checkpoint=False, **kwargs) -> None: +def load_model(path: str, driver_version="tpu_driver_nightly", hf_checkpoint=False, socketio_queue=None, initial_load=False, logger=None, **kwargs) -> None: global thread_resources_env, seq, tokenizer, network, params, pad_token_id if "pad_token_id" in kwargs: @@ -1195,6 +1195,7 @@ def encode(s, *args, **kwargs): thread_resources_env = maps.ResourceEnv(maps.Mesh(devices, ('dp', 'mp')), ()) maps.thread_resources.env = thread_resources_env + global shard_xmap, batch_xmap shard_xmap = __shard_xmap() batch_xmap = __batch_xmap(shard_dim=cores_per_replica) @@ -1244,6 +1245,7 @@ def encode(s, *args, **kwargs): from tqdm.auto import tqdm import functools + def callback(model_dict, f, **_): if callback.nested: return @@ -1317,19 +1319,20 @@ def callback(model_dict, f, **_): #if "no_transpose" not in transforms and tensor.ndim == 2: # tensor = tensor.T tensor.unsqueeze_(0) - if tensor.dtype is torch.float16 or tensor.dtype is torch.float32: - tensor = tensor.bfloat16() + # Shard the tensor so that parts of the tensor can be used # on different TPU cores + tensor = reshard_reverse( + tensor, + params["cores_per_replica"], + network.state["params"][spec["module"]][spec["param"]].shape, + ) + tensor = jnp.array(tensor.detach()) + if tensor.dtype is torch.float16 or tensor.dtype is torch.float32: + tensor = tensor.bfloat16() network.state["params"][spec["module"]][spec["param"]] = move_xmap( - jax.dlpack.from_dlpack(torch.utils.dlpack.to_dlpack( - reshard_reverse( - tensor, - params["cores_per_replica"], - network.state["params"][spec["module"]][spec["param"]].shape, - ) - )).copy(), + tensor, np.empty(params["cores_per_replica"]), ) @@ -1416,3 +1419,6 @@ def callback(model_dict, f, **_): model = GPTNeoForCausalLM.from_pretrained(vars.model, revision=vars.revision, cache_dir="cache") #network.state = network.move_xmap(network.state, np.zeros(cores_per_replica)) + global shard_xmap, batch_xmap + shard_xmap = __shard_xmap() + batch_xmap = __batch_xmap(shard_dim=cores_per_replica) \ No newline at end of file From 5f0e2001a7dc8cd630852450c7e4f9620d78a3f3 Mon Sep 17 00:00:00 2001 From: henk717 Date: Sun, 23 Apr 2023 17:50:03 +0200 Subject: [PATCH 28/35] Remove broken TPU disclaimer --- colab/TPU.ipynb | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/colab/TPU.ipynb b/colab/TPU.ipynb index 48fc7bdc8..20d0aba91 100644 --- a/colab/TPU.ipynb +++ b/colab/TPU.ipynb @@ -1,31 +1,8 @@ { "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "\"Open" - ] - }, { "cell_type": "markdown", "source": [ - "# GOOGLE HAS BROKEN SUPPORT FOR MESH TRANSFORMERS JAX IN THEIR DRIVER, THIS MESSAGE WILL BE REMOVED ONCE THE NOTEBOOK WORKS AGAIN\n", - "---\n", - "Are you a developer familair with Jax? We could use some help.\n", - "Our Mesh Transformers Jax fork resides here (but unfortunately VE-Forbryderne has gone missing) : https://github.com/VE-FORBRYDERNE/mesh-transformer-jax\n", - "\n", - "This is combined with the TPU backend code you can find here: https://github.com/KoboldAI/KoboldAI-Client/blob/main/tpu_mtj_backend.py\n", - "\n", - "So far we know the driver initialization issues can be resolved when a newer version of Jax is used combined with a slight edit to tpu_mtj_backend.py to use Jax's built in code for driver intialization (Which is not present in the older version we currently use, hence that part being in our file).\n", - "\n", - "As far as we understand the issue is that xmap was broken in newer versions, and MTJ makes use of it. If someone can port this part of the code to be compatible with the newer Jax versions you can save this notebook!\n", - "\n", - "(Or Google, if you are reading this. Please reintroduce the old 0.1 compatibility since you broke an entire ecosystem of Mesh Transformers Jax users, which has historic value because it is the original implementation of GPT-J. There also do not seem to be alternatives that have the same amount of performance and model compatibility).\n", - "\n", "# Welcome to KoboldAI on Google Colab, TPU Edition!\n", "KoboldAI is a powerful and easy way to use a variety of AI based text generation experiences. You can use it to write stories, blog posts, play a text adventure game, use it like a chatbot and more! In some cases it might even help you with an assignment or programming task (But always make sure the information the AI mentions is correct, it loves to make stuff up).\n", "\n", @@ -258,8 +235,7 @@ "colab": { "name": "ColabKobold TPU", "provenance": [], - "private_outputs": true, - "include_colab_link": true + "private_outputs": true }, "kernelspec": { "display_name": "Python 3", From b4cb09590f75e2843dfbb8aa39b106f475754a41 Mon Sep 17 00:00:00 2001 From: henk717 Date: Sun, 23 Apr 2023 18:23:38 +0200 Subject: [PATCH 29/35] Update requirements_mtj.txt --- requirements_mtj.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements_mtj.txt b/requirements_mtj.txt index ea399a8f8..41721424d 100644 --- a/requirements_mtj.txt +++ b/requirements_mtj.txt @@ -5,7 +5,6 @@ requests dm-haiku==0.0.9 jax==0.3.25 jaxlib==0.3.25 -transformers == 4.28.0 chex == 0.1.5 transformers == 4.24.0 huggingface_hub==0.12.1 From d88f10907367fd257fbe7111c1a535644cd19619 Mon Sep 17 00:00:00 2001 From: henk717 Date: Sun, 23 Apr 2023 18:49:25 +0200 Subject: [PATCH 30/35] TPU Fix Fix --- tpu_mtj_backend.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tpu_mtj_backend.py b/tpu_mtj_backend.py index f878d6909..205f9cc45 100644 --- a/tpu_mtj_backend.py +++ b/tpu_mtj_backend.py @@ -1195,11 +1195,6 @@ def encode(s, *args, **kwargs): thread_resources_env = maps.ResourceEnv(maps.Mesh(devices, ('dp', 'mp')), ()) maps.thread_resources.env = thread_resources_env - - global shard_xmap, batch_xmap - shard_xmap = __shard_xmap() - batch_xmap = __batch_xmap(shard_dim=cores_per_replica) - global badwords # These are the tokens that we don't want the AI to ever write badwords = jnp.array(vars.badwordsids).squeeze() @@ -1421,4 +1416,4 @@ def callback(model_dict, f, **_): #network.state = network.move_xmap(network.state, np.zeros(cores_per_replica)) global shard_xmap, batch_xmap shard_xmap = __shard_xmap() - batch_xmap = __batch_xmap(shard_dim=cores_per_replica) \ No newline at end of file + batch_xmap = __batch_xmap(shard_dim=cores_per_replica) From b808f039ab15a781b02b3bc00a8bbbf3624def31 Mon Sep 17 00:00:00 2001 From: henk717 Date: Sun, 23 Apr 2023 20:21:28 +0200 Subject: [PATCH 31/35] Pin TPU driver --- tpu_mtj_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpu_mtj_backend.py b/tpu_mtj_backend.py index 205f9cc45..fe20646c0 100644 --- a/tpu_mtj_backend.py +++ b/tpu_mtj_backend.py @@ -1049,7 +1049,7 @@ def read_neox_checkpoint(state, path, config, checkpoint_shards=2): raise RuntimeError(error) -def load_model(path: str, driver_version="tpu_driver_nightly", hf_checkpoint=False, socketio_queue=None, initial_load=False, logger=None, **kwargs) -> None: +def load_model(path: str, driver_version="tpu_driver_20221109", hf_checkpoint=False, socketio_queue=None, initial_load=False, logger=None, **kwargs) -> None: global thread_resources_env, seq, tokenizer, network, params, pad_token_id if "pad_token_id" in kwargs: From 1c36eb9f2cf438535a461c3e77b1d56bd324d0dc Mon Sep 17 00:00:00 2001 From: Henk Date: Sun, 23 Apr 2023 21:24:17 +0200 Subject: [PATCH 32/35] Further Isolation --- install_git_transformers.bat | 4 ++++ install_requirements.bat | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/install_git_transformers.bat b/install_git_transformers.bat index 3e154b800..8c6257cc9 100644 --- a/install_git_transformers.bat +++ b/install_git_transformers.bat @@ -1,6 +1,10 @@ @echo off cd /D %~dp0 + +:Isolation SET CONDA_SHLVL= +SET PYTHONNOUSERSITE=1 +SET PYTHONPATH= TITLE KoboldAI - Git Transformers Installer ECHO This script will replace the Transformers version with the latest Git Transformers which may contain breaking changes. diff --git a/install_requirements.bat b/install_requirements.bat index 2a4534c11..69e4a47a4 100644 --- a/install_requirements.bat +++ b/install_requirements.bat @@ -8,7 +8,11 @@ echo. Reg add "HKLM\SYSTEM\CurrentControlSet\Control\FileSystem" /v "LongPathsEnabled" /t REG_DWORD /d "1" /f 2>nul cd /D %~dp0 + +:Isolation SET CONDA_SHLVL= +SET PYTHONNOUSERSITE=1 +SET PYTHONPATH= if exist miniconda3\ ( echo Delete existing installation? From b8cd5f4c03e9bf2dd2936568d9abf2874775ed45 Mon Sep 17 00:00:00 2001 From: Henk Date: Sun, 23 Apr 2023 21:35:24 +0200 Subject: [PATCH 33/35] Revert "Added comregex_ui and comregex_ai to the UI under other settings. Needs proper titles and descriptions still." This reverts commit d16a3f4dc3746e65bc16e92d55b91dc15d7b3641. --- gensettings.py | 32 -------------------------------- koboldai_settings.py | 16 ++++------------ 2 files changed, 4 insertions(+), 44 deletions(-) diff --git a/gensettings.py b/gensettings.py index a0a20908e..8d68b4b53 100644 --- a/gensettings.py +++ b/gensettings.py @@ -881,38 +881,6 @@ "classname": "system", "name": "seed", "extra_classes": "var_sync_alt_system_seed_specified", - "ui_level": 2 - }, - { - "uitype": "text", - "unit": "text", - "label": "comregex_ai_string", - "id": "comregex_ai_string", - "min": 0, - "max": 1, - "step": 1, - "default": 1, - "tooltip": "Pattern for matching comments to remove them before sending them to the AI.", - "menu_path": "Settings", - "sub_path": "Other", - "classname": "system", - "name": "comregex_ai_string", - "ui_level": 2 - }, - { - "uitype": "text", - "unit": "text", - "label": "comregex_ui_string", - "id": "comregex_ui_string", - "min": 0, - "max": 1, - "step": 1, - "default": 1, - "tooltip": "Pattern for matching comments in the editor.", - "menu_path": "Settings", - "sub_path": "Other", - "classname": "system", - "name": "comregex_ui_string", "ui_level": 2 }, { diff --git a/koboldai_settings.py b/koboldai_settings.py index 407ae1e1c..06a15e4e8 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -1203,9 +1203,9 @@ def __setattr__(self, name, value): class system_settings(settings): local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', - 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'comregex_ui', + 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', - 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states'] + 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui'] no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', 'lua_koboldcore', 'sp', 'sp_length', '_horde_pid', 'horde_share', 'aibusy', 'serverstarted', 'inference_config', 'image_pipeline', 'summarizer', @@ -1252,10 +1252,8 @@ def __init__(self, socketio, koboldai_var): self.regex_sl = re.compile(r'\n*(?<=.) *\n(.|\n)*') # Pattern for limiting the output to a single line self.acregex_ai = re.compile(r'\n* *>(.|\n)*') # Pattern for matching adventure actions from the AI so we can remove them self.acregex_ui = re.compile(r'^ *(>.*)$', re.MULTILINE) # Pattern for matching actions in the HTML-escaped story so we can apply colouring, etc (make sure to encase part to format in parentheses) - self.comregex_ai_string = '(?:\n\[<\|(?:.|\n)*?\|>\](?=\n|$))|(?:\[<\|(?:.|\n)*?\|>\]\n?)' # Pattern for matching comments to remove them before sending them to the AI - self.comregex_ui_string = '(\[<\|(?:.|\n)*?\|>\])' # Pattern for matching comments in the editor - self.comregex_ai = re.compile(self.comregex_ai_string) # Pattern for matching comments to remove them before sending them to the AI - self.comregex_ui = re.compile(self.comregex_ui_string) # Pattern for matching comments in the editor + self.comregex_ai = re.compile(r'(?:\n\[<\|(?:.|\n)*?\|>\](?=\n|$))|(?:\[<\|(?:.|\n)*?\|>\]\n?)') # Pattern for matching comments to remove them before sending them to the AI + self.comregex_ui = re.compile(r'(\[<\|(?:.|\n)*?\|>\])') # Pattern for matching comments in the editor self.host = False self.flaskwebgui = False self.quiet = False # If set will suppress any story text from being printed to the console (will only be seen on the client web page) @@ -1342,12 +1340,6 @@ def __setattr__(self, name, value): self._socketio.emit('from_server', {'cmd': 'spstatitems', 'data': {self.spfilename: self.spmeta} if self.allowsp and len(self.spfilename) else {}}, namespace=None, broadcast=True, room="UI_1") super().__setattr__("sp_changed", False) - if name == 'comregex_ai_string': - self.comregex_ai = re.compile(self.comregex_ai_string) - - if name == 'comregex_ui_string': - self.comregex_ui = re.compile(self.comregex_ui_string) - if name == 'keep_img_gen_in_memory' and value == False: self.image_pipeline = None From 560c8d23fb20d97b486082e8a7193b109762e370 Mon Sep 17 00:00:00 2001 From: Henk Date: Sun, 23 Apr 2023 21:37:24 +0200 Subject: [PATCH 34/35] Re-add UI Json Fix --- koboldai_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koboldai_settings.py b/koboldai_settings.py index 06a15e4e8..dbc9d1150 100644 --- a/koboldai_settings.py +++ b/koboldai_settings.py @@ -1203,7 +1203,7 @@ def __setattr__(self, name, value): class system_settings(settings): local_only_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', - 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', + 'lua_koboldcore', 'regex_sl', 'acregex_ai', 'acregex_ui', 'comregex_ai', 'comregex_ui', 'sp', '_horde_pid', 'inference_config', 'image_pipeline', 'summarizer', 'summary_tokenizer', 'tts_model', 'rng_states', 'comregex_ai', 'comregex_ui'] no_save_variables = ['lua_state', 'lua_logname', 'lua_koboldbridge', 'lua_kobold', From 9eaa2aba47dda877fdf2a120bfbff9b1aa7e70ce Mon Sep 17 00:00:00 2001 From: Henk Date: Tue, 25 Apr 2023 22:49:56 +0200 Subject: [PATCH 35/35] Isolate OPT Tokenizer Fix to OPT models --- aiserver.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/aiserver.py b/aiserver.py index 9dd621ef3..7e198dee1 100644 --- a/aiserver.py +++ b/aiserver.py @@ -108,15 +108,14 @@ def new_init(self, *args, **kwargs): self.ncols = 99 tqdm.__init__ = new_init -# Fix some issues with the OPT tokenizer +# Add _koboldai_header support for some optional tokenizer fixes +# This used to be an OPT tokenizer fix, this has been moved search for "# These are model specific overrides if a model has bad defaults" for the new section from transformers import PreTrainedTokenizerBase old_pretrainedtokenizerbase_from_pretrained = PreTrainedTokenizerBase.from_pretrained.__func__ @classmethod def new_pretrainedtokenizerbase_from_pretrained(cls, *args, **kwargs): tokenizer = old_pretrainedtokenizerbase_from_pretrained(cls, *args, **kwargs) - tokenizer._koboldai_header = tokenizer.encode("") - tokenizer.add_bos_token = False - tokenizer.add_prefix_space = False + tokenizer._koboldai_header = [] return tokenizer PreTrainedTokenizerBase.from_pretrained = new_pretrainedtokenizerbase_from_pretrained @@ -3251,10 +3250,14 @@ def new_rebuild_tensor(storage: Union[torch_lazy_loader.LazyTensor, torch.Storag # koboldai_vars.badwordsids.append([vocab[key]]) # These are model specific overrides if a model has bad defaults + tokenizer._koboldai_header = [] if koboldai_vars.model_type == "llama": tokenizer.decode_with_prefix_space = True tokenizer.add_bos_token = False - + if koboldai_vars.model_type == "opt": + tokenizer._koboldai_header = tokenizer.encode("") + tokenizer.add_bos_token = False + tokenizer.add_prefix_space = False logger.info(f"Pipeline created: {koboldai_vars.model}") else: