From 4d698495eae6912db94dcdedb0c3b01c63143646 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 11:16:07 +0300 Subject: [PATCH 001/242] gguf : init --- .gitignore | 1 + Makefile | 7 +++++-- examples/gguf/gguf.cpp | 34 ++++++++++++++++++++++++++++++++++ ggml.h | 25 ++++++++++++++++++++++++- 4 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 examples/gguf/gguf.cpp diff --git a/.gitignore b/.gitignore index c1ab6bb6d08a3..abe8e28cb7687 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ models-mnt /server /Pipfile /embd-input-test +/gguf /libllama.so build-info.h arm_neon.h diff --git a/Makefile b/Makefile index fb7c27cd972bb..e19acfbb24a3f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Define the default target now so that it is always the first target -BUILD_TARGETS = main quantize quantize-stats perplexity embedding vdot train-text-from-scratch simple server embd-input-test +BUILD_TARGETS = main quantize quantize-stats perplexity embedding vdot train-text-from-scratch simple server embd-input-test gguf # Binaries only useful for tests TEST_TARGETS = tests/test-double-float tests/test-grad0 tests/test-opt tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0 @@ -330,7 +330,7 @@ libllama.so: llama.o ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -shared -fPIC -o $@ $^ $(LDFLAGS) clean: - rm -vf *.o *.so *.dll main quantize quantize-stats perplexity embedding benchmark-matmult save-load-state server simple vdot train-text-from-scratch embd-input-test build-info.h $(TEST_TARGETS) + rm -vf *.o *.so *.dll main quantize quantize-stats perplexity embedding benchmark-matmult save-load-state server simple vdot train-text-from-scratch embd-input-test gguf build-info.h $(TEST_TARGETS) # # Examples @@ -370,6 +370,9 @@ $(LIB_PRE)embdinput$(DSO_EXT): examples/embd-input/embd-input.h examples/embd-in embd-input-test: $(LIB_PRE)embdinput$(DSO_EXT) examples/embd-input/embd-input-test.cpp build-info.h ggml.o llama.o common.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %$(DSO_EXT),$(filter-out %.h,$(filter-out %.hpp,$^))) -o $@ $(LDFLAGS) -L. -lembdinput +gguf: examples/gguf/gguf.cpp build-info.h ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp build-info.h ggml.o llama.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp new file mode 100644 index 0000000000000..602de519adb1e --- /dev/null +++ b/examples/gguf/gguf.cpp @@ -0,0 +1,34 @@ +#include "ggml.h" + +#include +#include + +bool gguf_write(const std::string & fname) { + + + return true; +} + +bool gguf_read(const std::string & fname) { + return true; +} + +int main(int argc, char ** argv) { + if (argc < 3) { + fprintf(stdout, "usage: %s data.gguf r|w\n", argv[0]); + return -1; + } + + const std::string fname(argv[1]); + const std::string mode(argv[2]); + + GGML_ASSERT((mode == "r" || mode == "w") && "mode must be r or w"); + + if (mode == "w") { + GGML_ASSERT(gguf_write(fname) && "failed to write gguf file"); + } else if (mode == "r") { + GGML_ASSERT(gguf_read(fname) && "failed to read gguf file"); + } + + return 0; +} diff --git a/ggml.h b/ggml.h index 9919cce7c263f..2e700c9a06645 100644 --- a/ggml.h +++ b/ggml.h @@ -190,6 +190,9 @@ #define GGML_FILE_MAGIC 0x67676d6c // "ggml" #define GGML_FILE_VERSION 1 +#define GGUF_FILE_MAGIC 0x47475546 // "GGUF" +#define GGUF_FILE_VERSION 1 + #define GGML_QNT_VERSION 2 // bump this on quantization format changes #define GGML_QNT_VERSION_FACTOR 1000 // do not change this @@ -202,7 +205,6 @@ #define GGML_MAX_OP_PARAMS 32 #define GGML_DEFAULT_N_THREADS 4 - #define GGML_EXIT_SUCCESS 0 #define GGML_EXIT_ABORTED 1 @@ -1611,6 +1613,27 @@ extern "C" { GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist); + // + // gguf + // + + enum gguf_metadata_value_type { + GGUF_METADATA_VALUE_TYPE_UINT8 = 0, + GGUF_METADATA_VALUE_TYPE_INT8 = 1, + GGUF_METADATA_VALUE_TYPE_UINT16 = 2, + GGUF_METADATA_VALUE_TYPE_INT16 = 3, + GGUF_METADATA_VALUE_TYPE_UINT32 = 4, + GGUF_METADATA_VALUE_TYPE_INT32 = 5, + GGUF_METADATA_VALUE_TYPE_FLOAT32 = 6, + GGUF_METADATA_VALUE_TYPE_BOOL = 7, + GGUF_METADATA_VALUE_TYPE_STRING = 8, + GGUF_METADATA_VALUE_TYPE_ARRAY = 9, + }; + + struct gguf_string { + uint32_t n; + char * data; + }; // // system info // From bae6b125f6d6148d3bebb774f1b73ecc67dc0051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 26 Jul 2023 11:17:05 +0300 Subject: [PATCH 002/242] wip : implement GGUF (#2397) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add LLAMA_DEFAULT_RMS_EPS so we can change the default (#2384) Co-authored-by: Iwan Kawrakow * WIP: python class to write GGUF, incomplete C apı for reading --------- Co-authored-by: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Co-authored-by: Iwan Kawrakow --- constants.py | 32 +++++++ gguf.c | 192 ++++++++++++++++++++++++++++++++++++++ gguf.py | 257 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 481 insertions(+) create mode 100644 constants.py create mode 100644 gguf.c create mode 100644 gguf.py diff --git a/constants.py b/constants.py new file mode 100644 index 0000000000000..7c7456403aaeb --- /dev/null +++ b/constants.py @@ -0,0 +1,32 @@ +GGUF_MAGIC = 0x47475546 +GGUF_VERSION = 1 + +# general +KEY_GENERAL_ARCHITECTURE = "general.architecture" +KEY_GENERAL_QUANTIZATION_VERSION = "general.quantization_version" +KEY_GENERAL_NAME = "general.name" +KEY_GENERAL_AUTHOR = "general.author" +KEY_GENERAL_URL = "general.url" +KEY_GENERAL_DESCRIPTION = "general.description" +KEY_GENERAL_FILE_TYPE = "general.file_type" +KEY_GENERAL_LICENSE = "general.license" +KEY_GENERAL_SOURCE_URL = "general.source.url" +KEY_GENERAL_SOURCE_HF_REPO = "general.source.hugginface.repository" + +# LLM +KEY_LLM_CONTEXT_LENGTH = "{llm}.context_length" +KEY_LLM_EMBEDDING_LENGTH = "{llm}.embedding_length" +KEY_LLM_LAYER_COUNT = "{llm}.layer_count" +KEY_LLM_FEED_FORWARD_LENGTH = "{llm}.feed_forward_length" +KEY_LLM_USE_PARALLEL_RESIDUAL = "{llm}.use_parallel_residual" +KEY_LLM_TENSOR_DATA_LAYOUT = "{llm}.tensor_data_layout" + +# attention +KEY_ATTENTION_HEAD_COUNT = "{llm}.attention.head_count" +KEY_ATTENTION_HEAD_COUNT_KV = "{llm}.attention.head_count_kv" +KEY_ATTENTION_MAX_ALIBI_BIAS = "{llm}.attention.max_alibi_bias" +KEY_ATTENTION_CLAMP_KQV = "{llm}.attention.clamp_kqv" + +# RoPE +KEY_ROPE_DIMENSION_COUNT = "{llm}.rope.dimension_count" +KEY_ROPE_SCALE = "{llm}.rope.scale" diff --git a/gguf.c b/gguf.c new file mode 100644 index 0000000000000..54b31d411f35b --- /dev/null +++ b/gguf.c @@ -0,0 +1,192 @@ +// TODO: convert to proper gguf.h gguf.c structure, now I'm trying to be fast as much as possible, +// and everything is in this file for quick debugging. + +#include +#include +#include +#include + + +enum ggml_type { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_Q4_1 = 3, + // GGML_TYPE_Q4_2 = 4, support has been removed + // GGML_TYPE_Q4_3 (5) support has been removed + GGML_TYPE_Q5_0 = 6, + GGML_TYPE_Q5_1 = 7, + GGML_TYPE_Q8_0 = 8, + GGML_TYPE_Q8_1 = 9, + // k-quantizations + GGML_TYPE_Q2_K = 10, + GGML_TYPE_Q3_K = 11, + GGML_TYPE_Q4_K = 12, + GGML_TYPE_Q5_K = 13, + GGML_TYPE_Q6_K = 14, + GGML_TYPE_Q8_K = 15, + GGML_TYPE_I8, + GGML_TYPE_I16, + GGML_TYPE_I32, + GGML_TYPE_COUNT, +}; + +enum gguf_metadata_value_type { + GGUF_METADATA_VALUE_TYPE_UINT8 = 0, + GGUF_METADATA_VALUE_TYPE_INT8 = 1, + GGUF_METADATA_VALUE_TYPE_UINT16 = 2, + GGUF_METADATA_VALUE_TYPE_INT16 = 3, + GGUF_METADATA_VALUE_TYPE_UINT32 = 4, + GGUF_METADATA_VALUE_TYPE_INT32 = 5, + GGUF_METADATA_VALUE_TYPE_FLOAT32 = 6, + GGUF_METADATA_VALUE_TYPE_BOOL = 7, + GGUF_METADATA_VALUE_TYPE_STRING = 8, + GGUF_METADATA_VALUE_TYPE_ARRAY = 9, +}; + +struct gguf_string_t { + uint32_t len; + char * string; +}; + +union gguf_metadata_value_t; + +// Union definition for gguf_metadata_value_t +union gguf_metadata_value_t { + uint8_t uint8; + int8_t int8; + uint16_t uint16; + int16_t int16; + uint32_t uint32; + int32_t int32; + float float32; + bool bool_; + struct gguf_string_t string; + struct { + uint32_t len; + enum gguf_metadata_value_type type; + union gguf_metadata_value_t * array; + } array; +}; + + +struct gguf_metadata_kv_t { + struct gguf_string_t key; + uint32_t value_len; + enum gguf_metadata_value_type value_type; + union gguf_metadata_value_t* value; +}; + +struct gguf_header_t { + uint32_t magic; + uint32_t version; + uint32_t tensor_count; + uint32_t metadata_kv_count; + struct gguf_metadata_kv_t * metadata_kv; +}; + +struct gguf_tensor_info_t { + struct gguf_string_t name; + uint32_t n_dimensions; + uint32_t dimensions[]; +}; + +struct gguf_file_t { + struct gguf_header_t header; + uint8_t tensor_data[]; +}; + +void read_gguf_file(const char * file_path, struct gguf_file_t * gguf_file) { + FILE* file = fopen(file_path, "rb"); + if (file == NULL) { + printf("Error opening the file.\n"); + return; + } + + fread(&gguf_file->header.magic, sizeof(uint32_t), 1, file); + + // Verify magic and version + if (gguf_file->header.magic != 0x47475546) { + printf("Invalid magic number. Not a valid GGUF file.\n"); + fclose(file); + return; + } + + fread(&gguf_file->header.version, sizeof(uint32_t), 1, file); + + if (gguf_file->header.version != 1) { + printf("Unsupported version. Expected version 1.\n"); + fclose(file); + return; + } + + fread(&gguf_file->header.tensor_count, sizeof(uint32_t), 1, file); + fread(&gguf_file->header.metadata_kv_count, sizeof(uint32_t), 1, file); + + printf("Magic: %x\n", gguf_file->header.magic); + printf("Version: %d\n", gguf_file->header.version); + printf("Tensor Count: %d\n", gguf_file->header.tensor_count); + printf("Metadata Key-Value Count: %d\n", gguf_file->header.metadata_kv_count); + + gguf_file->header.metadata_kv = (struct gguf_metadata_kv_t*)malloc(gguf_file->header.metadata_kv_count * sizeof(struct gguf_metadata_kv_t)); + + for (int i = 0; i < gguf_file->header.metadata_kv_count; i++) { + struct gguf_metadata_kv_t* kv = &gguf_file->header.metadata_kv[i]; + fread(&kv->key.len, sizeof(uint32_t), 1, file); + kv->key.string = (char*)malloc(kv->key.len ); // Allocate memory for the key string + fread(kv->key.string, sizeof(char), kv->key.len, file); + //kv->key.string[kv->key.len] = '\0'; // Null-terminate the key string + + fread(&kv->value_type, sizeof(uint32_t), 1, file); + + printf("Metadata Value Type: %d\n", kv->value_type); + printf("Metadata Key: %s\n", kv->key.string); + + // Read metadata value according to its type using reinterpret_cast + switch (kv->value_type) { + case GGUF_METADATA_VALUE_TYPE_UINT32: + kv->value = (uint32_t *) malloc(sizeof(uint32_t)); + fread(kv->value, sizeof(uint32_t), 1, file); + printf("value: %d\n", kv->value->uint32); + break; + case GGUF_METADATA_VALUE_TYPE_FLOAT32: + kv->value = (float *)malloc(sizeof(float)); + fread(kv->value, sizeof(float), 1, file); + printf("value: %f\n", (float)kv->value->float32); + break; + case GGUF_METADATA_VALUE_TYPE_STRING: + fread(&kv->value_len, sizeof(uint32_t), 1, file); + printf("value len: %d\n", kv->value_len); +kv->value = (char *)malloc(sizeof(char) * kv->value_len); // Allocate memory for the value string +fread(kv->value, sizeof(char), kv->value_len, file); + printf("value: %s\n", (char *)kv->value); + break; + // ... (handle other types in a similar manner) + default: + printf("Unsupported metadata value type.\n"); + fclose(file); + return; + } + } + + // TODO: handle reading tensor data + + fclose(file); +} + +void gguf_free(struct gguf_file_t * gguf_file) { + // Free allocated memory for key strings avd values + for (int i = 0; i < gguf_file->header.metadata_kv_count; i++) { + free(gguf_file->header.metadata_kv[i].key.string); + free(gguf_file->header.metadata_kv[i].value); + } + free(gguf_file->header.metadata_kv); +} + +int main() { + const char* file_path = "example.gguf"; + struct gguf_file_t gguf_file; + read_gguf_file(file_path, &gguf_file); + gguf_free(&gguf_file); + return 0; +} diff --git a/gguf.py b/gguf.py new file mode 100644 index 0000000000000..dfd5ba5bf8490 --- /dev/null +++ b/gguf.py @@ -0,0 +1,257 @@ +"""TODOs +1. Implement writing tensor data with alignment. +2. Implement writers for known architectures, LLaMA in particular. +3. Add docstrings from the format specs. +4. After development is done, Convert it to a proper pip-installable Python package, and possibly move it to its own repo under ggml-org. +""" + +import struct +from enum import IntEnum +from typing import List, Any +import constants + + +class GGMLQuantizationType(IntEnum): + F32 = 0 + F16 = 1 + QR_0 = 2 + Q4_1 = 3 + # Q4_2 = 4 # support has been removed + # Q4_3 = 5 # support has been removed + Q5_0 = 6 + Q5_1 = 7 + Q8_0 = 8 + Q8_1 = 9 + Q2_K = 10 + Q3_K = 11 + Q4_K = 12 + Q5_K = 13 + Q6_K = 14 + Q8_K = 15 + + +class GGUFValueType(IntEnum): + UINT8 = 0 + INT8 = 1 + UINT16 = 2 + INT16 = 3 + UINT32 = 4 + INT32 = 5 + FLOAT32 = 6 + BOOL = 7 + STRING = 8 + ARRAY = 9 + + @staticmethod + def get_type(value): + if isinstance(value, str): + return GGUFValueType.STRING + elif isinstance(value, list): + return GGUFValueType.ARRAY + elif isinstance(value, float): + return GGUFValueType.FLOAT32 + elif isinstance(value, bool): + return GGUFValueType.BOOL + else: + return GGUFValueType.INT32 + + +class GGUFWriter: + def __init__(self, buffered_writer): + self.buffered_writer = buffered_writer + + def write_header(self, tensor_count: int, metadata_kv_count: int): + self.buffered_writer.write(struct.pack(" "GGUFWriter": + f = open(path, "wb") + return cls(f) + + def write_key(self, key: str, value_type: GGUFValueType): + encoded_key = key.encode("utf8") + self.buffered_writer.write(struct.pack(" Date: Wed, 26 Jul 2023 11:26:14 +0300 Subject: [PATCH 003/242] ci : disable CI temporary to not waste energy --- .github/ISSUE_TEMPLATE/custom.md | 185 --------- .github/workflows/build.yml | 632 ----------------------------- .github/workflows/docker.yml | 65 --- .github/workflows/editorconfig.yml | 17 - .github/workflows/tidy-post.yml | 20 - .github/workflows/tidy-review.yml | 23 -- 6 files changed, 942 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/custom.md delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/editorconfig.yml delete mode 100644 .github/workflows/tidy-post.yml delete mode 100644 .github/workflows/tidy-review.yml diff --git a/.github/ISSUE_TEMPLATE/custom.md b/.github/ISSUE_TEMPLATE/custom.md deleted file mode 100644 index 8fd9553567780..0000000000000 --- a/.github/ISSUE_TEMPLATE/custom.md +++ /dev/null @@ -1,185 +0,0 @@ ---- -name: Issue and enhancement template -about: Used to report issues and request enhancements for llama.cpp -title: "[User] Insert summary of your issue or enhancement.." -labels: '' -assignees: '' - ---- - -# Prerequisites - -Please answer the following questions for yourself before submitting an issue. - -- [ ] I am running the latest code. Development is very rapid so there are no tagged versions as of now. -- [ ] I carefully followed the [README.md](https://github.com/ggerganov/llama.cpp/blob/master/README.md). -- [ ] I [searched using keywords relevant to my issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/filtering-and-searching-issues-and-pull-requests) to make sure that I am creating a new issue that is not already open (or closed). -- [ ] I reviewed the [Discussions](https://github.com/ggerganov/llama.cpp/discussions), and have a new bug or useful enhancement to share. - -# Expected Behavior - -Please provide a detailed written description of what you were trying to do, and what you expected `llama.cpp` to do. - -# Current Behavior - -Please provide a detailed written description of what `llama.cpp` did, instead. - -# Environment and Context - -Please provide detailed information about your computer setup. This is important in case the issue is not reproducible except for under certain specific conditions. - -* Physical (or virtual) hardware you are using, e.g. for Linux: - -`$ lscpu` - -* Operating System, e.g. for Linux: - -`$ uname -a` - -* SDK version, e.g. for Linux: - -``` -$ python3 --version -$ make --version -$ g++ --version -``` - -# Failure Information (for bugs) - -Please help provide information about the failure if this is a bug. If it is not a bug, please remove the rest of this template. - -# Steps to Reproduce - -Please provide detailed steps for reproducing the issue. We are not sitting in front of your screen, so the more detail the better. - -1. step 1 -2. step 2 -3. step 3 -4. etc. - -# Failure Logs - -Please include any relevant log snippets or files. If it works under one configuration but not under another, please provide logs for both configurations and their corresponding outputs so it is easy to see where behavior changes. - -Also, please try to **avoid using screenshots** if at all possible. Instead, copy/paste the console output and use [Github's markdown](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax) to cleanly format your logs for easy readability. - -Example environment info: -``` -llama.cpp$ git log | head -1 -commit 2af23d30434a677c6416812eea52ccc0af65119c - -llama.cpp$ lscpu | egrep "AMD|Flags" -Vendor ID: AuthenticAMD -Model name: AMD Ryzen Threadripper 1950X 16-Core Processor -Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid amd_dcm aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb hw_pstate ssbd ibpb vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt sha_ni xsaveopt xsavec xgetbv1 xsaves clzero irperf xsaveerptr arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif overflow_recov succor smca sme sev -Virtualization: AMD-V - -llama.cpp$ python3 --version -Python 3.10.9 - -llama.cpp$ pip list | egrep "torch|numpy|sentencepiece" -numpy 1.24.2 -numpydoc 1.5.0 -sentencepiece 0.1.97 -torch 1.13.1 -torchvision 0.14.1 - -llama.cpp$ make --version | head -1 -GNU Make 4.3 - -$ md5sum ./models/65B/ggml-model-q4_0.bin -dbdd682cce80e2d6e93cefc7449df487 ./models/65B/ggml-model-q4_0.bin -``` - -Example run with the Linux command [perf](https://www.brendangregg.com/perf.html) -``` -llama.cpp$ perf stat ./main -m ./models/65B/ggml-model-q4_0.bin -t 16 -n 1024 -p "Please close your issue when it has been answered." -main: seed = 1679149377 -llama_model_load: loading model from './models/65B/ggml-model-q4_0.bin' - please wait ... -llama_model_load: n_vocab = 32000 -llama_model_load: n_ctx = 512 -llama_model_load: n_embd = 8192 -llama_model_load: n_mult = 256 -llama_model_load: n_head = 64 -llama_model_load: n_layer = 80 -llama_model_load: n_rot = 128 -llama_model_load: f16 = 2 -llama_model_load: n_ff = 22016 -llama_model_load: n_parts = 8 -llama_model_load: ggml ctx size = 41477.73 MB -llama_model_load: memory_size = 2560.00 MB, n_mem = 40960 -llama_model_load: loading model part 1/8 from './models/65B/ggml-model-q4_0.bin' -llama_model_load: .......................................................................................... done -llama_model_load: model size = 4869.09 MB / num tensors = 723 -llama_model_load: loading model part 2/8 from './models/65B/ggml-model-q4_0.bin.1' -llama_model_load: .......................................................................................... done -llama_model_load: model size = 4869.09 MB / num tensors = 723 -llama_model_load: loading model part 3/8 from './models/65B/ggml-model-q4_0.bin.2' -llama_model_load: .......................................................................................... done -llama_model_load: model size = 4869.09 MB / num tensors = 723 -llama_model_load: loading model part 4/8 from './models/65B/ggml-model-q4_0.bin.3' -llama_model_load: .......................................................................................... done -llama_model_load: model size = 4869.09 MB / num tensors = 723 -llama_model_load: loading model part 5/8 from './models/65B/ggml-model-q4_0.bin.4' -llama_model_load: .......................................................................................... done -llama_model_load: model size = 4869.09 MB / num tensors = 723 -llama_model_load: loading model part 6/8 from './models/65B/ggml-model-q4_0.bin.5' -llama_model_load: .......................................................................................... done -llama_model_load: model size = 4869.09 MB / num tensors = 723 -llama_model_load: loading model part 7/8 from './models/65B/ggml-model-q4_0.bin.6' -llama_model_load: .......................................................................................... done -llama_model_load: model size = 4869.09 MB / num tensors = 723 -llama_model_load: loading model part 8/8 from './models/65B/ggml-model-q4_0.bin.7' -llama_model_load: .......................................................................................... done -llama_model_load: model size = 4869.09 MB / num tensors = 723 - -system_info: n_threads = 16 / 32 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | VSX = 0 | - -main: prompt: 'Please close your issue when it has been answered.' -main: number of tokens in prompt = 11 - 1 -> '' - 12148 -> 'Please' - 3802 -> ' close' - 596 -> ' your' - 2228 -> ' issue' - 746 -> ' when' - 372 -> ' it' - 756 -> ' has' - 1063 -> ' been' - 7699 -> ' answered' - 29889 -> '.' - -sampling parameters: temp = 0.800000, top_k = 40, top_p = 0.950000, repeat_last_n = 64, repeat_penalty = 1.300000 - - -Please close your issue when it has been answered. -@duncan-donut: I'm trying to figure out what kind of "support" you need for this script and why, exactly? Is there a question about how the code works that hasn't already been addressed in one or more comments below this ticket, or are we talking something else entirely like some sorta bugfixing job because your server setup is different from mine?? -I can understand if your site needs to be running smoothly and you need help with a fix of sorts but there should really be nothing wrong here that the code itself could not handle. And given that I'm getting reports about how it works perfectly well on some other servers, what exactly are we talking? A detailed report will do wonders in helping us get this resolved for ya quickly so please take your time and describe the issue(s) you see as clearly & concisely as possible!! -@duncan-donut: I'm not sure if you have access to cPanel but you could try these instructions. It is worth a shot! Let me know how it goes (or what error message, exactly!) when/if ya give that code a go? [end of text] - - -main: mem per token = 71159620 bytes -main: load time = 19309.95 ms -main: sample time = 168.62 ms -main: predict time = 223895.61 ms / 888.47 ms per token -main: total time = 246406.42 ms - - Performance counter stats for './main -m ./models/65B/ggml-model-q4_0.bin -t 16 -n 1024 -p Please close your issue when it has been answered.': - - 3636882.89 msec task-clock # 14.677 CPUs utilized - 13509 context-switches # 3.714 /sec - 2436 cpu-migrations # 0.670 /sec - 10476679 page-faults # 2.881 K/sec - 13133115082869 cycles # 3.611 GHz (16.77%) - 29314462753 stalled-cycles-frontend # 0.22% frontend cycles idle (16.76%) - 10294402631459 stalled-cycles-backend # 78.39% backend cycles idle (16.74%) - 23479217109614 instructions # 1.79 insn per cycle - # 0.44 stalled cycles per insn (16.76%) - 2353072268027 branches # 647.002 M/sec (16.77%) - 1998682780 branch-misses # 0.08% of all branches (16.76%) - - 247.802177522 seconds time elapsed - - 3618.573072000 seconds user - 18.491698000 seconds sys -``` diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 84faad37ab95a..0000000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,632 +0,0 @@ -name: CI - -on: - workflow_dispatch: # allows manual triggering - inputs: - create_release: - description: 'Create new release' - required: true - type: boolean - push: - branches: - - master - paths: ['.github/workflows/**', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu'] - pull_request: - types: [opened, synchronize, reopened] - paths: ['**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu'] - -env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - GGML_NLOOP: 3 - GGML_NITER: 1 - GGML_N_THREADS: 1 - -jobs: - ubuntu-focal-make: - runs-on: ubuntu-20.04 - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v1 - - - name: Dependencies - id: depends - run: | - sudo apt-get update - sudo apt-get install build-essential gcc-8 - - - name: Build - id: make_build - run: | - CC=gcc-8 make - - ubuntu-latest-cmake: - runs-on: ubuntu-latest - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v1 - - - name: Dependencies - id: depends - run: | - sudo apt-get update - sudo apt-get install build-essential - - - name: Build - id: cmake_build - run: | - mkdir build - cd build - cmake .. - cmake --build . --config Release - - - name: Test - id: cmake_test - run: | - cd build - ctest --verbose --timeout 900 - - ubuntu-latest-cmake-sanitizer: - runs-on: ubuntu-latest - - continue-on-error: true - - strategy: - matrix: - sanitizer: [ADDRESS, THREAD, UNDEFINED] - build_type: [Debug, Release] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v1 - - - name: Dependencies - id: depends - run: | - sudo apt-get update - sudo apt-get install build-essential - - - name: Build - id: cmake_build - run: | - mkdir build - cd build - cmake .. -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - cmake --build . --config ${{ matrix.build_type }} - - - name: Test - id: cmake_test - run: | - cd build - ctest --verbose --timeout 900 - - ubuntu-latest-cmake-mpi: - runs-on: ubuntu-latest - - continue-on-error: true - - strategy: - matrix: - mpi_library: [mpich, libopenmpi-dev] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v1 - - - name: Dependencies - id: depends - run: | - sudo apt-get update - sudo apt-get install build-essential ${{ matrix.mpi_library }} - - - name: Build - id: cmake_build - run: | - mkdir build - cd build - cmake -DLLAMA_MPI=ON .. - cmake --build . --config Release - - - name: Test - id: cmake_test - run: | - cd build - ctest --verbose - - macOS-latest-make: - runs-on: macos-latest - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v1 - - - name: Dependencies - id: depends - continue-on-error: true - run: | - brew update - - - name: Build - id: make_build - run: | - make - - macOS-latest-cmake: - runs-on: macos-latest - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v1 - - - name: Dependencies - id: depends - continue-on-error: true - run: | - brew update - - - name: Build - id: cmake_build - run: | - sysctl -a - mkdir build - cd build - cmake -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF .. - cmake --build . --config Release - - - name: Test - id: cmake_test - run: | - cd build - ctest --verbose --timeout 900 - - windows-latest-cmake: - runs-on: windows-latest - - env: - OPENBLAS_VERSION: 0.3.23 - OPENCL_VERSION: 2023.04.17 - CLBLAST_VERSION: 1.6.0 - - strategy: - matrix: - include: - - build: 'noavx' - defines: '-DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF' - - build: 'avx2' - defines: '-DLLAMA_BUILD_SERVER=ON' - - build: 'avx' - defines: '-DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX2=OFF' - - build: 'avx512' - defines: '-DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX512=ON -DBUILD_SHARED_LIBS=ON' - - build: 'clblast' - defines: '-DLLAMA_BUILD_SERVER=ON -DLLAMA_CLBLAST=ON -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/clblast"' - - build: 'openblas' - defines: '-DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"' - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v1 - - - name: Download OpenCL SDK - id: get_opencl - if: ${{ matrix.build == 'clblast' }} - run: | - curl.exe -o $env:RUNNER_TEMP/opencl.zip -L "https://github.com/KhronosGroup/OpenCL-SDK/releases/download/v${env:OPENCL_VERSION}/OpenCL-SDK-v${env:OPENCL_VERSION}-Win-x64.zip" - mkdir $env:RUNNER_TEMP/opencl - tar.exe -xvf $env:RUNNER_TEMP/opencl.zip --strip-components=1 -C $env:RUNNER_TEMP/opencl - - - name: Download CLBlast - id: get_clblast - if: ${{ matrix.build == 'clblast' }} - run: | - curl.exe -o $env:RUNNER_TEMP/clblast.7z -L "https://github.com/CNugteren/CLBlast/releases/download/${env:CLBLAST_VERSION}/CLBlast-${env:CLBLAST_VERSION}-windows-x64.7z" - curl.exe -o $env:RUNNER_TEMP/CLBlast.LICENSE.txt -L "https://github.com/CNugteren/CLBlast/raw/${env:CLBLAST_VERSION}/LICENSE" - 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/clblast.7z - rename-item $env:RUNNER_TEMP/CLBlast-${env:CLBLAST_VERSION}-windows-x64 clblast - foreach ($f in (gci -Recurse -Path "$env:RUNNER_TEMP/clblast" -Filter '*.cmake')) { - $txt = Get-Content -Path $f -Raw - $txt.Replace('C:/vcpkg/packages/opencl_x64-windows/', "$($env:RUNNER_TEMP.Replace('\','/'))/opencl/") | Set-Content -Path $f -Encoding UTF8 - } - - - name: Download OpenBLAS - id: get_openblas - if: ${{ matrix.build == 'openblas' }} - run: | - curl.exe -o $env:RUNNER_TEMP/openblas.zip -L "https://github.com/xianyi/OpenBLAS/releases/download/v${env:OPENBLAS_VERSION}/OpenBLAS-${env:OPENBLAS_VERSION}-x64.zip" - curl.exe -o $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt -L "https://github.com/xianyi/OpenBLAS/raw/v${env:OPENBLAS_VERSION}/LICENSE" - mkdir $env:RUNNER_TEMP/openblas - tar.exe -xvf $env:RUNNER_TEMP/openblas.zip -C $env:RUNNER_TEMP/openblas - $vcdir = $(vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath) - $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim())) - $lib = $(join-path $msvc 'bin\Hostx64\x64\lib.exe') - & $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll - - - name: Build - id: cmake_build - run: | - mkdir build - cd build - cmake .. ${{ matrix.defines }} - cmake --build . --config Release - - - name: Add clblast.dll - id: add_clblast_dll - if: ${{ matrix.build == 'clblast' }} - run: | - cp $env:RUNNER_TEMP/clblast/lib/clblast.dll ./build/bin/Release - cp $env:RUNNER_TEMP/CLBlast.LICENSE.txt ./build/bin/Release/CLBlast-${env:CLBLAST_VERSION}.txt - - - name: Add libopenblas.dll - id: add_libopenblas_dll - if: ${{ matrix.build == 'openblas' }} - run: | - cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll - cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt - - - name: Check AVX512F support - id: check_avx512f - if: ${{ matrix.build == 'avx512' }} - continue-on-error: true - run: | - cd build - $vcdir = $(vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath) - $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim())) - $cl = $(join-path $msvc 'bin\Hostx64\x64\cl.exe') - echo 'int main(void){unsigned int a[4];__cpuid(a,7);return !(a[1]&65536);}' >> avx512f.c - & $cl /O2 /GS- /kernel avx512f.c /link /nodefaultlib /entry:main - .\avx512f.exe && echo "AVX512F: YES" && ( echo HAS_AVX512F=1 >> $env:GITHUB_ENV ) || echo "AVX512F: NO" - - - name: Test - id: cmake_test - if: ${{ matrix.build != 'clblast' && (matrix.build != 'avx512' || env.HAS_AVX512F == '1') }} # Test AVX-512 only when possible - run: | - cd build - ctest -C Release --verbose --timeout 900 - - - name: Get commit hash - id: commit - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: pr-mpt/actions-commit-hash@v2 - - - name: Pack artifacts - id: pack_artifacts - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - run: | - Copy-Item LICENSE .\build\bin\Release\llama.cpp.txt - 7z a llama-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip .\build\bin\Release\* - - - name: Upload artifacts - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: actions/upload-artifact@v3 - with: - path: | - llama-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip - - windows-latest-cmake-cublas: - runs-on: windows-latest - - strategy: - matrix: - cuda: ['12.1.0', '11.7.1'] - build: ['cublas'] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v1 - - - uses: Jimver/cuda-toolkit@v0.2.10 - id: cuda-toolkit - with: - cuda: ${{ matrix.cuda }} - # TODO(green-sky): _dev seems to fail, and non dev are not enought - #sub-packages: '["nvcc", "cudart", "cublas", "cudart_dev", "cublas_dev"]' - - - name: Build - id: cmake_build - run: | - mkdir build - cd build - cmake .. -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUBLAS=ON - cmake --build . --config Release - - - name: Get commit hash - id: commit - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: pr-mpt/actions-commit-hash@v2 - - - name: Pack artifacts - id: pack_artifacts - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - run: | - 7z a llama-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-cu${{ matrix.cuda }}-x64.zip .\build\bin\Release\* - - - name: Upload artifacts - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: actions/upload-artifact@v3 - with: - path: | - llama-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-cu${{ matrix.cuda }}-x64.zip - - - name: Copy and pack Cuda runtime - if: ${{ matrix.cuda == '12.1.0' }} - # TODO(green-sky): paths are cuda 12 specific - run: | - echo "Cuda install location: ${{steps.cuda-toolkit.outputs.CUDA_PATH}}" - mkdir '.\build\bin\cudart\' - cp "${{steps.cuda-toolkit.outputs.CUDA_PATH}}\bin\cudart64_12.dll" '.\build\bin\cudart\' - cp "${{steps.cuda-toolkit.outputs.CUDA_PATH}}\bin\cublas64_12.dll" '.\build\bin\cudart\' - cp "${{steps.cuda-toolkit.outputs.CUDA_PATH}}\bin\cublasLt64_12.dll" '.\build\bin\cudart\' - 7z a cudart-llama-bin-win-cu${{ matrix.cuda }}-x64.zip .\build\bin\cudart\* - - - name: Copy and pack Cuda runtime - if: ${{ matrix.cuda == '11.7.1' }} - # TODO(green-sky): paths are cuda 11 specific - run: | - echo "Cuda install location: ${{steps.cuda-toolkit.outputs.CUDA_PATH}}" - mkdir '.\build\bin\cudart\' - ls "${{steps.cuda-toolkit.outputs.CUDA_PATH}}\bin" - cp "${{steps.cuda-toolkit.outputs.CUDA_PATH}}\bin\cudart64_110.dll" '.\build\bin\cudart\' - cp "${{steps.cuda-toolkit.outputs.CUDA_PATH}}\bin\cublas64_11.dll" '.\build\bin\cudart\' - cp "${{steps.cuda-toolkit.outputs.CUDA_PATH}}\bin\cublasLt64_11.dll" '.\build\bin\cudart\' - 7z a cudart-llama-bin-win-cu${{ matrix.cuda }}-x64.zip .\build\bin\cudart\* - - - name: Upload Cuda runtime - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: actions/upload-artifact@v3 - with: - path: | - cudart-llama-bin-win-cu${{ matrix.cuda }}-x64.zip - - release: - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - - runs-on: ubuntu-latest - - needs: - - ubuntu-focal-make - - ubuntu-latest-cmake - - macOS-latest-make - - macOS-latest-cmake - - windows-latest-cmake - - windows-latest-cmake-cublas - - steps: - - name: Download artifacts - id: download-artifact - uses: actions/download-artifact@v3 - - - name: Get commit hash - id: commit - uses: pr-mpt/actions-commit-hash@v2 - - - name: Create release - id: create_release - uses: anzz1/action-create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }} - - - name: Upload release - id: upload_release - uses: actions/github-script@v3 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const path = require('path'); - const fs = require('fs'); - const release_id = '${{ steps.create_release.outputs.id }}'; - for (let file of await fs.readdirSync('./artifact')) { - if (path.extname(file) === '.zip') { - console.log('uploadReleaseAsset', file); - await github.repos.uploadReleaseAsset({ - owner: context.repo.owner, - repo: context.repo.repo, - release_id: release_id, - name: file, - data: await fs.readFileSync(`./artifact/${file}`) - }); - } - } - -# ubuntu-latest-gcc: -# runs-on: ubuntu-latest -# -# strategy: -# matrix: -# build: [Debug, Release] -# -# steps: -# - name: Clone -# uses: actions/checkout@v1 -# -# - name: Dependencies -# run: | -# sudo apt-get update -# sudo apt-get install build-essential -# sudo apt-get install cmake -# -# - name: Configure -# run: cmake . -DCMAKE_BUILD_TYPE=${{ matrix.build }} -# -# - name: Build -# run: | -# make -# -# ubuntu-latest-clang: -# runs-on: ubuntu-latest -# -# strategy: -# matrix: -# build: [Debug, Release] -# -# steps: -# - name: Clone -# uses: actions/checkout@v1 -# -# - name: Dependencies -# run: | -# sudo apt-get update -# sudo apt-get install build-essential -# sudo apt-get install cmake -# -# - name: Configure -# run: cmake . -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -# -# - name: Build -# run: | -# make -# -# ubuntu-latest-gcc-sanitized: -# runs-on: ubuntu-latest -# -# strategy: -# matrix: -# sanitizer: [ADDRESS, THREAD, UNDEFINED] -# -# steps: -# - name: Clone -# uses: actions/checkout@v1 -# -# - name: Dependencies -# run: | -# sudo apt-get update -# sudo apt-get install build-essential -# sudo apt-get install cmake -# -# - name: Configure -# run: cmake . -DCMAKE_BUILD_TYPE=Debug -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON -# -# - name: Build -# run: | -# make -# -# windows: -# runs-on: windows-latest -# -# strategy: -# matrix: -# build: [Release] -# arch: [Win32, x64] -# include: -# - arch: Win32 -# s2arc: x86 -# - arch: x64 -# s2arc: x64 -# -# steps: -# - name: Clone -# uses: actions/checkout@v1 -# -# - name: Add msbuild to PATH -# uses: microsoft/setup-msbuild@v1 -# -# - name: Configure -# run: > -# cmake -S . -B ./build -A ${{ matrix.arch }} -# -DCMAKE_BUILD_TYPE=${{ matrix.build }} -# -# - name: Build -# run: | -# cd ./build -# msbuild ALL_BUILD.vcxproj -t:build -p:configuration=${{ matrix.build }} -p:platform=${{ matrix.arch }} -# -# - name: Upload binaries -# uses: actions/upload-artifact@v1 -# with: -# name: llama-bin-${{ matrix.arch }} -# path: build/bin/${{ matrix.build }} -# -# windows-blas: -# runs-on: windows-latest -# -# strategy: -# matrix: -# build: [Release] -# arch: [Win32, x64] -# blas: [ON] -# include: -# - arch: Win32 -# obzip: https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x86.zip -# s2arc: x86 -# - arch: x64 -# obzip: https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip -# s2arc: x64 -# -# steps: -# - name: Clone -# uses: actions/checkout@v1 -# -# - name: Add msbuild to PATH -# uses: microsoft/setup-msbuild@v1 -# -# - name: Fetch OpenBLAS -# if: matrix.blas == 'ON' -# run: | -# C:/msys64/usr/bin/wget.exe -qO blas.zip ${{ matrix.obzip }} -# 7z x blas.zip -oblas -y -# copy blas/include/cblas.h . -# copy blas/include/openblas_config.h . -# echo "blasdir=$env:GITHUB_WORKSPACE/blas" >> $env:GITHUB_ENV -# -# - name: Configure -# run: > -# cmake -S . -B ./build -A ${{ matrix.arch }} -# -DCMAKE_BUILD_TYPE=${{ matrix.build }} -# -DLLAMA_SUPPORT_OPENBLAS=${{ matrix.blas }} -# -DCMAKE_LIBRARY_PATH="$env:blasdir/lib" -# -# - name: Build -# run: | -# cd ./build -# msbuild ALL_BUILD.vcxproj -t:build -p:configuration=${{ matrix.build }} -p:platform=${{ matrix.arch }} -# -# - name: Copy libopenblas.dll -# if: matrix.blas == 'ON' -# run: copy "$env:blasdir/bin/libopenblas.dll" build/bin/${{ matrix.build }} -# -# - name: Upload binaries -# if: matrix.blas == 'ON' -# uses: actions/upload-artifact@v1 -# with: -# name: llama-blas-bin-${{ matrix.arch }} -# path: build/bin/${{ matrix.build }} -# -# emscripten: -# runs-on: ubuntu-latest -# -# strategy: -# matrix: -# build: [Release] -# -# steps: -# - name: Clone -# uses: actions/checkout@v1 -# -# - name: Dependencies -# run: | -# wget -q https://github.com/emscripten-core/emsdk/archive/master.tar.gz -# tar -xvf master.tar.gz -# emsdk-master/emsdk update -# emsdk-master/emsdk install latest -# emsdk-master/emsdk activate latest -# -# - name: Configure -# run: echo "tmp" -# -# - name: Build -# run: | -# pushd emsdk-master -# source ./emsdk_env.sh -# popd -# emcmake cmake . -DCMAKE_BUILD_TYPE=${{ matrix.build }} -# make diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 379fbd7ad35f1..0000000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,65 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -# GitHub recommends pinning actions to a commit SHA. -# To get a newer version, you will need to update the SHA. -# You can also reference a tag or branch, but the action may change without warning. - -name: Publish Docker image - -on: - pull_request: - push: - branches: - - master - -jobs: - push_to_registry: - name: Push Docker image to Docker Hub - if: github.event.pull_request.draft == false - - runs-on: ubuntu-latest - env: - COMMIT_SHA: ${{ github.sha }} - strategy: - matrix: - config: - - { tag: "light", dockerfile: ".devops/main.Dockerfile" } - - { tag: "full", dockerfile: ".devops/full.Dockerfile" } - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push Docker image (versioned) - if: github.event_name == 'push' - uses: docker/build-push-action@v4 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - tags: "ghcr.io/ggerganov/llama.cpp:${{ matrix.config.tag }}-${{ env.COMMIT_SHA }}" - file: ${{ matrix.config.dockerfile }} - - - name: Build and push Docker image (tagged) - uses: docker/build-push-action@v4 - with: - context: . - push: ${{ github.event_name == 'push' }} - platforms: linux/amd64,linux/arm64 - tags: "ghcr.io/ggerganov/llama.cpp:${{ matrix.config.tag }}" - file: ${{ matrix.config.dockerfile }} diff --git a/.github/workflows/editorconfig.yml b/.github/workflows/editorconfig.yml deleted file mode 100644 index b4e535acf1f64..0000000000000 --- a/.github/workflows/editorconfig.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: EditorConfig Checker - -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - editorconfig: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: editorconfig-checker/action-editorconfig-checker@main - - run: editorconfig-checker diff --git a/.github/workflows/tidy-post.yml b/.github/workflows/tidy-post.yml deleted file mode 100644 index 03652760c80dc..0000000000000 --- a/.github/workflows/tidy-post.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: clang-tidy review post comments - -on: - workflow_dispatch: - workflows: ["clang-tidy-review"] - types: - - completed - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: ZedThree/clang-tidy-review/post@v0.13.0 - # lgtm_comment_body, max_comments, and annotations need to be set on the posting workflow in a split setup - with: - # adjust options as necessary - lgtm_comment_body: '' - annotations: false - max_comments: 25 diff --git a/.github/workflows/tidy-review.yml b/.github/workflows/tidy-review.yml deleted file mode 100644 index a4bc8d976560e..0000000000000 --- a/.github/workflows/tidy-review.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: clang-tidy-review - -on: - pull_request: - branches: - - master - -jobs: - clang-tidy-review: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - uses: ZedThree/clang-tidy-review@v0.13.0 - id: review - with: - lgtm_comment_body: '' - build_dir: build - cmake_command: cmake . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=on - split_workflow: true - - - uses: ZedThree/clang-tidy-review/upload@v0.13.0 From 6873148771a08861922fb5070b18b56bc25f3701 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 13:24:20 +0300 Subject: [PATCH 004/242] gguf : first API pass --- ggml.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ggml.h | 55 +++++++++++++++++++++++++++++++++------------- 2 files changed, 109 insertions(+), 15 deletions(-) diff --git a/ggml.c b/ggml.c index 33459f263657e..8c01dcabf1671 100644 --- a/ggml.c +++ b/ggml.c @@ -18297,6 +18297,75 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i //////////////////////////////////////////////////////////////////////////////// +struct gguf_string { + uint32_t n; + char * data; +}; + +union gguf_value; + +union gguf_value { + uint8_t uint8; + int8_t int8; + uint16_t uint16; + int16_t int16; + uint32_t uint32; + int32_t int32; + float float32; + bool bool_; + + struct gguf_string str; + + struct { + enum gguf_type type; + + uint32_t n; + union gguf_value * arr; + } arr; +}; + +struct gguf_kv { + struct gguf_string key; + + uint32_t n_bytes; // TODO: is this actually needed? + + enum gguf_type type; + union gguf_value value; +}; + +struct gguf_header { + uint32_t magic; + uint32_t version; + uint32_t n_tensors; + + uint32_t n_kv; + struct gguf_kv * kv; +}; + +struct gguf_tensor_info { + struct gguf_string name; + + uint32_t n_dims; + uint32_t ne[GGML_MAX_DIMS]; + uint32_t n_elements; // TODO: is this needed? + + enum ggml_type type; + + uint64_t offset; // must be a multiple of `ALIGNMENT`. +}; + +struct gguf_context { + struct gguf_header header; + struct gguf_tensor_info * infos; + + size_t alignment; + + uint8_t * padding; + uint8_t * data; +}; + +//////////////////////////////////////////////////////////////////////////////// + int ggml_cpu_has_avx(void) { #if defined(__AVX__) return 1; diff --git a/ggml.h b/ggml.h index 2e700c9a06645..51885917fab69 100644 --- a/ggml.h +++ b/ggml.h @@ -204,6 +204,7 @@ #define GGML_MAX_NAME 48 #define GGML_MAX_OP_PARAMS 32 #define GGML_DEFAULT_N_THREADS 4 +#define GGUF_DEFAULT_ALIGNMENT 32 #define GGML_EXIT_SUCCESS 0 #define GGML_EXIT_ABORTED 1 @@ -1617,23 +1618,47 @@ extern "C" { // gguf // - enum gguf_metadata_value_type { - GGUF_METADATA_VALUE_TYPE_UINT8 = 0, - GGUF_METADATA_VALUE_TYPE_INT8 = 1, - GGUF_METADATA_VALUE_TYPE_UINT16 = 2, - GGUF_METADATA_VALUE_TYPE_INT16 = 3, - GGUF_METADATA_VALUE_TYPE_UINT32 = 4, - GGUF_METADATA_VALUE_TYPE_INT32 = 5, - GGUF_METADATA_VALUE_TYPE_FLOAT32 = 6, - GGUF_METADATA_VALUE_TYPE_BOOL = 7, - GGUF_METADATA_VALUE_TYPE_STRING = 8, - GGUF_METADATA_VALUE_TYPE_ARRAY = 9, + enum gguf_type { + GGUF_TYPE_UINT8 = 0, + GGUF_TYPE_INT8 = 1, + GGUF_TYPE_UINT16 = 2, + GGUF_TYPE_INT16 = 3, + GGUF_TYPE_UINT32 = 4, + GGUF_TYPE_INT32 = 5, + GGUF_TYPE_FLOAT32 = 6, + GGUF_TYPE_BOOL = 7, + GGUF_TYPE_STRING = 8, + GGUF_TYPE_ARRAY = 9, }; - struct gguf_string { - uint32_t n; - char * data; - }; + struct gguf_context; + + GGML_API struct gguf_context * gguf_gguf_init(const char * path); + GGML_API void gguf_gguf_free(struct gguf_context * ctx); + + GGML_API int gguf_get_version (struct gguf_context * ctx); + GGML_API size_t gguf_get_alignment (struct gguf_context * ctx); + GGML_API size_t gguf_get_data_offset(struct gguf_context * ctx); + + GGML_API int gguf_get_n_kv(struct gguf_context * ctx); + GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); + GGML_API enum gguf_type gguf_get_type(struct gguf_context * ctx, int i); + GGML_API void gguf_get_val (struct gguf_context * ctx, int i, void * val); + + GGML_API uint8_t gguf_get_val_u8 (struct gguf_context * ctx, int i); + GGML_API int8_t gguf_get_val_i8 (struct gguf_context * ctx, int i); + GGML_API uint16_t gguf_get_val_u16 (struct gguf_context * ctx, int i); + GGML_API int16_t gguf_get_val_i16 (struct gguf_context * ctx, int i); + GGML_API uint32_t gguf_get_val_u32 (struct gguf_context * ctx, int i); + GGML_API int32_t gguf_get_val_i32 (struct gguf_context * ctx, int i); + GGML_API float gguf_get_val_f32 (struct gguf_context * ctx, int i); + GGML_API bool gguf_get_val_bool(struct gguf_context * ctx, int i); + GGML_API const char * gguf_get_val_str (struct gguf_context * ctx, int i); + // TODO: arr + + GGML_API int gguf_get_n_tensors (struct gguf_context * ctx); + GGML_API size_t gguf_get_tensor_offset(struct gguf_context * ctx, int i); + // // system info // From 8d6acfec127bcf769ee015cb005ad12c02631cfb Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 14:33:53 +0300 Subject: [PATCH 005/242] gguf : read header + meta data --- ggml.c | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- ggml.h | 13 ++++--- 2 files changed, 120 insertions(+), 11 deletions(-) diff --git a/ggml.c b/ggml.c index 8c01dcabf1671..0c46518025b95 100644 --- a/ggml.c +++ b/ggml.c @@ -18297,7 +18297,7 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i //////////////////////////////////////////////////////////////////////////////// -struct gguf_string { +struct gguf_str { uint32_t n; char * data; }; @@ -18314,7 +18314,7 @@ union gguf_value { float float32; bool bool_; - struct gguf_string str; + struct gguf_str str; struct { enum gguf_type type; @@ -18325,7 +18325,7 @@ union gguf_value { }; struct gguf_kv { - struct gguf_string key; + struct gguf_str key; uint32_t n_bytes; // TODO: is this actually needed? @@ -18337,13 +18337,13 @@ struct gguf_header { uint32_t magic; uint32_t version; uint32_t n_tensors; - uint32_t n_kv; + struct gguf_kv * kv; }; struct gguf_tensor_info { - struct gguf_string name; + struct gguf_str name; uint32_t n_dims; uint32_t ne[GGML_MAX_DIMS]; @@ -18364,6 +18364,114 @@ struct gguf_context { uint8_t * data; }; +static bool gguf_fread_el(void * dst, size_t size, FILE * file, size_t * offset) { + const size_t n = fread(dst, 1, size, file); + *offset += n; + return n == size; +} + +static bool gguf_fread_str(void * dst, FILE * file, size_t * offset) { + struct gguf_str * p = (struct gguf_str *) dst; + + p->n = 0; + p->data = NULL; + + bool ok = true; + + ok = ok && gguf_fread_el(&p->n, sizeof(p->n), file, offset); + ok = ok && gguf_fread_el(&p->data, p->n, file, offset); + + return ok; +} + +struct gguf_context * gguf_init(const char * path, bool load) { + FILE * file = fopen(path, "rb"); + if (!file) { + return NULL; + } + + // offset from start of file + size_t offset = 0; + + // check the magic before making allocations + uint32_t magic = 0; + gguf_fread_el(&magic, sizeof(magic), file, &offset); + if (magic != GGUF_MAGIC) { + fprintf(stderr, "gguf: invalid magic number %08x\n", magic); + fclose(file); + return NULL; + } + + bool ok = true; + + struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context)); + + ctx->header.magic = magic; + + ok = ok && gguf_fread_el(&ctx->header.version, sizeof(ctx->header.version), file, &offset); + ok = ok && gguf_fread_el(&ctx->header.n_tensors, sizeof(ctx->header.n_tensors), file, &offset); + ok = ok && gguf_fread_el(&ctx->header.n_kv, sizeof(ctx->header.n_kv), file, &offset); + + if (!ok) { + fprintf(stderr, "gguf: failed to read header\n"); + fclose(file); + gguf_free(ctx); + return NULL; + } + + ctx->header.kv = GGML_ALIGNED_MALLOC(ctx->header.n_kv * sizeof(struct gguf_kv)); + + for (uint32_t i = 0; i < ctx->header.n_kv; ++i) { + struct gguf_kv * kv = &ctx->header.kv[i]; + + ok = ok && gguf_fread_str(&kv->key, file, &offset); + //ok = ok && gguf_fread_el (&kv->n_bytes, sizeof(kv->n_bytes), file, &offset); + ok = ok && gguf_fread_el (&kv->type, sizeof(kv->type), file, &offset); + + switch (kv->type) { + case GGUF_TYPE_UINT8: + ok = ok && gguf_fread_el (&kv->value.uint8, sizeof(kv->value.uint8), file, &offset); break; + case GGUF_TYPE_INT8: + ok = ok && gguf_fread_el (&kv->value.int8, sizeof(kv->value.int8), file, &offset); break; + case GGUF_TYPE_UINT16: + ok = ok && gguf_fread_el (&kv->value.uint16, sizeof(kv->value.uint16), file, &offset); break; + case GGUF_TYPE_INT16: + ok = ok && gguf_fread_el (&kv->value.int16, sizeof(kv->value.int16), file, &offset); break; + case GGUF_TYPE_UINT32: + ok = ok && gguf_fread_el (&kv->value.uint32, sizeof(kv->value.uint32), file, &offset); break; + case GGUF_TYPE_INT32: + ok = ok && gguf_fread_el (&kv->value.int32, sizeof(kv->value.int32), file, &offset); break; + case GGUF_TYPE_FLOAT32: + ok = ok && gguf_fread_el (&kv->value.float32, sizeof(kv->value.float32), file, &offset); break; + case GGUF_TYPE_BOOL: + ok = ok && gguf_fread_el (&kv->value.bool_, sizeof(kv->value.bool_), file, &offset); break; + case GGUF_TYPE_STRING: + ok = ok && gguf_fread_str(&kv->value.str, file, &offset); break; + case GGUF_TYPE_ARRAY: + GGML_ASSERT("gguf: array type not implemented"); + break; + }; + } + + if (!ok) { + fprintf(stderr, "gguf: failed to read key-value pairs\n"); + free(ctx->header.kv); + fclose(file); + gguf_free(ctx); + return NULL; + } + + ctx->alignment = GGUF_DEFAULT_ALIGNMENT; + + + + return ctx; +} + +void gguf_free(struct gguf_context * ctx) { + GGML_ALIGNED_FREE(ctx); +} + //////////////////////////////////////////////////////////////////////////////// int ggml_cpu_has_avx(void) { diff --git a/ggml.h b/ggml.h index 51885917fab69..1983bcd3e46fb 100644 --- a/ggml.h +++ b/ggml.h @@ -190,9 +190,6 @@ #define GGML_FILE_MAGIC 0x67676d6c // "ggml" #define GGML_FILE_VERSION 1 -#define GGUF_FILE_MAGIC 0x47475546 // "GGUF" -#define GGUF_FILE_VERSION 1 - #define GGML_QNT_VERSION 2 // bump this on quantization format changes #define GGML_QNT_VERSION_FACTOR 1000 // do not change this @@ -204,11 +201,15 @@ #define GGML_MAX_NAME 48 #define GGML_MAX_OP_PARAMS 32 #define GGML_DEFAULT_N_THREADS 4 -#define GGUF_DEFAULT_ALIGNMENT 32 #define GGML_EXIT_SUCCESS 0 #define GGML_EXIT_ABORTED 1 +#define GGUF_MAGIC 0x47475546 // "GGUF" +#define GGUF_VERSION 1 + +#define GGUF_DEFAULT_ALIGNMENT 32 + #define GGML_UNUSED(x) (void)(x) #define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1)) @@ -1633,8 +1634,8 @@ extern "C" { struct gguf_context; - GGML_API struct gguf_context * gguf_gguf_init(const char * path); - GGML_API void gguf_gguf_free(struct gguf_context * ctx); + GGML_API struct gguf_context * gguf_init(const char * path, bool load); + GGML_API void gguf_free(struct gguf_context * ctx); GGML_API int gguf_get_version (struct gguf_context * ctx); GGML_API size_t gguf_get_alignment (struct gguf_context * ctx); From d91b985d2db894d9194506e0fdc2d50c31371777 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 14:58:35 +0300 Subject: [PATCH 006/242] gguf : read tensor info --- ggml.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/ggml.c b/ggml.c index 0c46518025b95..4c30725fe2c88 100644 --- a/ggml.c +++ b/ggml.c @@ -18347,7 +18347,7 @@ struct gguf_tensor_info { uint32_t n_dims; uint32_t ne[GGML_MAX_DIMS]; - uint32_t n_elements; // TODO: is this needed? + uint32_t n_elms; // TODO: is this needed? enum ggml_type type; @@ -18359,8 +18359,9 @@ struct gguf_context { struct gguf_tensor_info * infos; size_t alignment; + size_t offset; - uint8_t * padding; + //uint8_t * padding; uint8_t * data; }; @@ -18461,9 +18462,55 @@ struct gguf_context * gguf_init(const char * path, bool load) { return NULL; } + ctx->infos = GGML_ALIGNED_MALLOC(ctx->header.n_tensors * sizeof(struct gguf_tensor_info)); + + for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { + struct gguf_tensor_info * info = &ctx->infos[i]; + + memset(info->ne, 0, sizeof(info->ne)); + + ok = ok && gguf_fread_str(&info->name, file, &offset); + ok = ok && gguf_fread_el (&info->n_dims, sizeof(info->n_dims), file, &offset); + for (uint32_t j = 0; j < info->n_dims; ++j) { + ok = ok && gguf_fread_el (&info->ne[j], sizeof(info->ne[j]), file, &offset); + } + //ok = ok && gguf_fread_el (&info->n_elms, sizeof(info->n_elms), file, &offset); + ok = ok && gguf_fread_el (&info->type, sizeof(info->type), file, &offset); + ok = ok && gguf_fread_el (&info->offset, sizeof(info->offset), file, &offset); + + if (!ok) { + fprintf(stderr, "gguf: failed to read tensor info\n"); + free(ctx->header.kv); + free(ctx->infos); + fclose(file); + gguf_free(ctx); + return NULL; + } + } + ctx->alignment = GGUF_DEFAULT_ALIGNMENT; + // TODO: determine new alignment from kv if available + { + const size_t offset_pad = offset % ctx->alignment; + + if (offset_pad != 0) { + offset += ctx->alignment - offset_pad; + fseek(file, offset, SEEK_SET); + } + } + + ctx->offset = offset; + + if (load) { + GGML_ASSERT("gguf: load not implemented"); + // - compute total tensor size + // - allocate buffer + // - read tensor data into buffer + // - add gguf_get_tensor_data() API + // - maybe create a ggml_context and return it + } return ctx; } From 78b226a9597c662a81b5ba986f64fb42b8de40eb Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 16:32:05 +0300 Subject: [PATCH 007/242] gguf : initial model loading - not tested --- ggml.c | 327 ++++++++++++++++++++++++++++++++++++++++++++++++++------- ggml.h | 13 ++- 2 files changed, 301 insertions(+), 39 deletions(-) diff --git a/ggml.c b/ggml.c index 4c30725fe2c88..f252363d960a3 100644 --- a/ggml.c +++ b/ggml.c @@ -18351,7 +18351,7 @@ struct gguf_tensor_info { enum ggml_type type; - uint64_t offset; // must be a multiple of `ALIGNMENT`. + uint64_t offset; // offset from beginning of file, must be a multiple of `ALIGNMENT` }; struct gguf_context { @@ -18359,7 +18359,8 @@ struct gguf_context { struct gguf_tensor_info * infos; size_t alignment; - size_t offset; + size_t offset; // offset of `data` from beginning of file + size_t size_data; // size of `data` in bytes //uint8_t * padding; uint8_t * data; @@ -18379,14 +18380,15 @@ static bool gguf_fread_str(void * dst, FILE * file, size_t * offset) { bool ok = true; - ok = ok && gguf_fread_el(&p->n, sizeof(p->n), file, offset); + // TODO: how to avoid mallocs for strings? + ok = ok && gguf_fread_el(&p->n, sizeof(p->n), file, offset); p->data = calloc(p->n + 1, 1); ok = ok && gguf_fread_el(&p->data, p->n, file, offset); return ok; } -struct gguf_context * gguf_init(const char * path, bool load) { - FILE * file = fopen(path, "rb"); +struct gguf_context * gguf_init(const char * fname, struct gguf_init_params params) { + FILE * file = fopen(fname, "rb"); if (!file) { return NULL; } @@ -18398,7 +18400,7 @@ struct gguf_context * gguf_init(const char * path, bool load) { uint32_t magic = 0; gguf_fread_el(&magic, sizeof(magic), file, &offset); if (magic != GGUF_MAGIC) { - fprintf(stderr, "gguf: invalid magic number %08x\n", magic); + fprintf(stderr, "%s: invalid magic number %08x\n", __func__, magic); fclose(file); return NULL; } @@ -18408,13 +18410,17 @@ struct gguf_context * gguf_init(const char * path, bool load) { struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context)); ctx->header.magic = magic; + ctx->header.kv = NULL; + + ctx->infos = NULL; + ctx->data = NULL; ok = ok && gguf_fread_el(&ctx->header.version, sizeof(ctx->header.version), file, &offset); ok = ok && gguf_fread_el(&ctx->header.n_tensors, sizeof(ctx->header.n_tensors), file, &offset); ok = ok && gguf_fread_el(&ctx->header.n_kv, sizeof(ctx->header.n_kv), file, &offset); if (!ok) { - fprintf(stderr, "gguf: failed to read header\n"); + fprintf(stderr, "%s: failed to read header\n", __func__); fclose(file); gguf_free(ctx); return NULL; @@ -18430,24 +18436,15 @@ struct gguf_context * gguf_init(const char * path, bool load) { ok = ok && gguf_fread_el (&kv->type, sizeof(kv->type), file, &offset); switch (kv->type) { - case GGUF_TYPE_UINT8: - ok = ok && gguf_fread_el (&kv->value.uint8, sizeof(kv->value.uint8), file, &offset); break; - case GGUF_TYPE_INT8: - ok = ok && gguf_fread_el (&kv->value.int8, sizeof(kv->value.int8), file, &offset); break; - case GGUF_TYPE_UINT16: - ok = ok && gguf_fread_el (&kv->value.uint16, sizeof(kv->value.uint16), file, &offset); break; - case GGUF_TYPE_INT16: - ok = ok && gguf_fread_el (&kv->value.int16, sizeof(kv->value.int16), file, &offset); break; - case GGUF_TYPE_UINT32: - ok = ok && gguf_fread_el (&kv->value.uint32, sizeof(kv->value.uint32), file, &offset); break; - case GGUF_TYPE_INT32: - ok = ok && gguf_fread_el (&kv->value.int32, sizeof(kv->value.int32), file, &offset); break; - case GGUF_TYPE_FLOAT32: - ok = ok && gguf_fread_el (&kv->value.float32, sizeof(kv->value.float32), file, &offset); break; - case GGUF_TYPE_BOOL: - ok = ok && gguf_fread_el (&kv->value.bool_, sizeof(kv->value.bool_), file, &offset); break; - case GGUF_TYPE_STRING: - ok = ok && gguf_fread_str(&kv->value.str, file, &offset); break; + case GGUF_TYPE_UINT8: ok = ok && gguf_fread_el (&kv->value.uint8, sizeof(kv->value.uint8), file, &offset); break; + case GGUF_TYPE_INT8: ok = ok && gguf_fread_el (&kv->value.int8, sizeof(kv->value.int8), file, &offset); break; + case GGUF_TYPE_UINT16: ok = ok && gguf_fread_el (&kv->value.uint16, sizeof(kv->value.uint16), file, &offset); break; + case GGUF_TYPE_INT16: ok = ok && gguf_fread_el (&kv->value.int16, sizeof(kv->value.int16), file, &offset); break; + case GGUF_TYPE_UINT32: ok = ok && gguf_fread_el (&kv->value.uint32, sizeof(kv->value.uint32), file, &offset); break; + case GGUF_TYPE_INT32: ok = ok && gguf_fread_el (&kv->value.int32, sizeof(kv->value.int32), file, &offset); break; + case GGUF_TYPE_FLOAT32: ok = ok && gguf_fread_el (&kv->value.float32, sizeof(kv->value.float32), file, &offset); break; + case GGUF_TYPE_BOOL: ok = ok && gguf_fread_el (&kv->value.bool_, sizeof(kv->value.bool_), file, &offset); break; + case GGUF_TYPE_STRING: ok = ok && gguf_fread_str(&kv->value.str, file, &offset); break; case GGUF_TYPE_ARRAY: GGML_ASSERT("gguf: array type not implemented"); break; @@ -18455,8 +18452,7 @@ struct gguf_context * gguf_init(const char * path, bool load) { } if (!ok) { - fprintf(stderr, "gguf: failed to read key-value pairs\n"); - free(ctx->header.kv); + fprintf(stderr, "%s: failed to read key-value pairs\n", __func__); fclose(file); gguf_free(ctx); return NULL; @@ -18467,7 +18463,7 @@ struct gguf_context * gguf_init(const char * path, bool load) { for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { struct gguf_tensor_info * info = &ctx->infos[i]; - memset(info->ne, 0, sizeof(info->ne)); + memset(info->ne, 1, sizeof(info->ne)); ok = ok && gguf_fread_str(&info->name, file, &offset); ok = ok && gguf_fread_el (&info->n_dims, sizeof(info->n_dims), file, &offset); @@ -18479,9 +18475,7 @@ struct gguf_context * gguf_init(const char * path, bool load) { ok = ok && gguf_fread_el (&info->offset, sizeof(info->offset), file, &offset); if (!ok) { - fprintf(stderr, "gguf: failed to read tensor info\n"); - free(ctx->header.kv); - free(ctx->infos); + fprintf(stderr, "%s: failed to read tensor info\n", __func__); fclose(file); gguf_free(ctx); return NULL; @@ -18503,22 +18497,279 @@ struct gguf_context * gguf_init(const char * path, bool load) { ctx->offset = offset; - if (load) { - GGML_ASSERT("gguf: load not implemented"); - // - compute total tensor size - // - allocate buffer - // - read tensor data into buffer - // - add gguf_get_tensor_data() API - // - maybe create a ggml_context and return it + ctx->size_data = 0; + + for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { + struct gguf_tensor_info * info = &ctx->infos[i]; + + const int64_t ne = + (int64_t) info->ne[0] * + (int64_t) info->ne[1] * + (int64_t) info->ne[2] * + (int64_t) info->ne[3]; + + if (ne % ggml_blck_size(info->type) != 0) { + fprintf(stderr, "%s: tensor '%s' number of elements (%" PRId64 ") is not a multiple of block size (%d)\n", + __func__, info->name.data, ne, ggml_blck_size(info->type)); + fclose(file); + gguf_free(ctx); + return NULL; + } + + const size_t size_cur = (ne*ggml_type_size(info->type))/ggml_blck_size(info->type); + + // TODO: pad size_cur to alignment + ctx->size_data += size_cur; + } + + // TODO: simplify + if (params.load) { + if (params.malloc) { + ctx->data = GGML_ALIGNED_MALLOC(ctx->size_data); + fseek(file, ctx->offset, SEEK_SET); + ok = ok && gguf_fread_el(ctx->data, ctx->size_data, file, &offset); + } else if (params.ctx != NULL) { + bool ctx_new = false; + bool ctx_no_alloc = false; + + if (*params.ctx == NULL) { + const size_t mem_size = + ctx->header.n_tensors*ggml_tensor_overhead() + 1 + + ctx->size_data; + + struct ggml_init_params pdata = { + .mem_size = mem_size, + .mem_buffer = NULL, + .no_alloc = false, + }; + + *params.ctx = ggml_init(pdata); + + ctx_new = true; + } else { + ctx_no_alloc = ggml_get_no_alloc(*params.ctx); + ggml_set_no_alloc(*params.ctx, false); + } + + struct ggml_context * ctx_data = *params.ctx; + + struct ggml_tensor * data = ggml_new_tensor_1d(ctx_data, GGML_TYPE_I8, ctx->size_data); + + // read the tensor data + ok = ok && gguf_fread_el(data->data, ctx->size_data, file, &offset); + + if (!ok) { + fprintf(stderr, "%s: failed to read tensor data\n", __func__); + fclose(file); + if (ctx_new) { + ggml_free(ctx_data); + } else { + ggml_set_no_alloc(ctx_data, ctx_no_alloc); + } + gguf_free(ctx); + return NULL; + } + + ctx->data = data->data; + + // create the tensors + ggml_set_no_alloc(ctx_data, true); + + for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { + const int64_t ne[GGML_MAX_DIMS] = { + ctx->infos[i].ne[0], + ctx->infos[i].ne[1], + ctx->infos[i].ne[2], + ctx->infos[i].ne[3], + }; + + struct ggml_tensor * cur = ggml_new_tensor(ctx_data, ctx->infos[i].type, ctx->infos[i].n_dims, ne); + + ok = ok && cur != NULL; + + if (!ok) { + break; + } + + cur->data = (char *) data->data + ctx->infos[i].offset - ctx->offset; + } + + if (!ok) { + fprintf(stderr, "%s: failed to create tensors\n", __func__); + fclose(file); + if (ctx_new) { + ggml_free(ctx_data); + } else { + ggml_set_no_alloc(ctx_data, ctx_no_alloc); + } + gguf_free(ctx); + return NULL; + } + + ggml_set_no_alloc(ctx_data, ctx_no_alloc); + } else { + GGML_ASSERT("gguf: invalid params - load requires malloc or ctx"); + } + } + + if (!ok) { + fprintf(stderr, "%s: failed to read tensor data\n", __func__); + fclose(file); + gguf_free(ctx); + return NULL; } return ctx; } void gguf_free(struct gguf_context * ctx) { + if (ctx == NULL) { + return; + } + + if (ctx->header.kv) { + // free string memory - not great.. + for (uint32_t i = 0; i < ctx->header.n_kv; ++i) { + struct gguf_kv * kv = &ctx->header.kv[i]; + + if (kv->key.data) { + free(kv->key.data); + } + + if (kv->type == GGUF_TYPE_STRING) { + if (kv->value.str.data) { + free(kv->value.str.data); + } + } + } + + GGML_ALIGNED_FREE(ctx->header.kv); + } + + if (ctx->infos) { + for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { + struct gguf_tensor_info * info = &ctx->infos[i]; + + if (info->name.data) { + free(info->name.data); + } + } + + GGML_ALIGNED_FREE(ctx->infos); + } + GGML_ALIGNED_FREE(ctx); } +int gguf_get_version(struct gguf_context * ctx) { + return ctx->header.version; +} + +size_t gguf_get_alignment(struct gguf_context * ctx) { + return ctx->alignment; +} + +size_t gguf_get_data_offset(struct gguf_context * ctx) { + return ctx->offset; +} + +void * gguf_get_data(struct gguf_context * ctx) { + return ctx->data; +} + +int gguf_get_n_kv(struct gguf_context * ctx) { + return ctx->header.n_kv; +} + +const char * gguf_get_key(struct gguf_context * ctx, int i) { + return ctx->header.kv[i].key.data; +} + +enum gguf_type gguf_get_type(struct gguf_context * ctx, int i) { + return ctx->header.kv[i].type; +} + +void gguf_get_val(struct gguf_context * ctx, int i, void * val) { + struct gguf_kv * kv = &ctx->header.kv[i]; + + switch (kv->type) { + case GGUF_TYPE_UINT8: memcpy(val, &kv->value.uint8, sizeof(uint8_t)); break; + case GGUF_TYPE_INT8: memcpy(val, &kv->value.int8, sizeof(int8_t)); break; + case GGUF_TYPE_UINT16: memcpy(val, &kv->value.uint16, sizeof(uint16_t)); break; + case GGUF_TYPE_INT16: memcpy(val, &kv->value.int16, sizeof(int16_t)); break; + case GGUF_TYPE_UINT32: memcpy(val, &kv->value.uint32, sizeof(uint32_t)); break; + case GGUF_TYPE_INT32: memcpy(val, &kv->value.int32, sizeof(int32_t)); break; + case GGUF_TYPE_FLOAT32: memcpy(val, &kv->value.float32, sizeof(float)); break; + case GGUF_TYPE_BOOL: memcpy(val, &kv->value.bool_, sizeof(bool)); break; + case GGUF_TYPE_STRING: memcpy(val, &kv->value.str.data, sizeof(char *)); break; + default: + GGML_ASSERT("gguf: not implemented"); + break; + } +} + +uint8_t gguf_get_val_u8(struct gguf_context * ctx, int i) { + uint8_t val; + gguf_get_val(ctx, i, &val); + return val; +} + +int8_t gguf_get_val_i8(struct gguf_context * ctx, int i) { + int8_t val; + gguf_get_val(ctx, i, &val); + return val; +} + +uint16_t gguf_get_val_u16(struct gguf_context * ctx, int i) { + uint16_t val; + gguf_get_val(ctx, i, &val); + return val; +} + +int16_t gguf_get_val_i16(struct gguf_context * ctx, int i) { + int16_t val; + gguf_get_val(ctx, i, &val); + return val; +} + +uint32_t gguf_get_val_u32(struct gguf_context * ctx, int i) { + uint32_t val; + gguf_get_val(ctx, i, &val); + return val; +} + +int32_t gguf_get_val_i32(struct gguf_context * ctx, int i) { + int32_t val; + gguf_get_val(ctx, i, &val); + return val; +} + +float gguf_get_val_f32(struct gguf_context * ctx, int i) { + float val; + gguf_get_val(ctx, i, &val); + return val; +} + +bool gguf_get_val_bool(struct gguf_context * ctx, int i) { + bool val; + gguf_get_val(ctx, i, &val); + return val; +} + +const char * gguf_get_val_str (struct gguf_context * ctx, int i) { + char * val; + gguf_get_val(ctx, i, &val); + return val; +} + +int gguf_get_n_tensors(struct gguf_context * ctx) { + return ctx->header.n_tensors; +} + +size_t gguf_get_tensor_offset(struct gguf_context * ctx, int i) { + return ctx->infos[i].offset; +} + //////////////////////////////////////////////////////////////////////////////// int ggml_cpu_has_avx(void) { diff --git a/ggml.h b/ggml.h index 1983bcd3e46fb..fac0f5e687e23 100644 --- a/ggml.h +++ b/ggml.h @@ -1634,12 +1634,23 @@ extern "C" { struct gguf_context; - GGML_API struct gguf_context * gguf_init(const char * path, bool load); + struct gguf_init_params { + bool load; // load the tensor data + bool malloc; // if false, use the provided ggml_context to allocate the tensor data + // it no ggml_context is provided, it will be created + // if true, use malloc to allocate the tensor data + + struct ggml_context ** ctx; + }; + + GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); + //GGML_API struct gguf_context * gguf_init_from_buffer(..); GGML_API void gguf_free(struct gguf_context * ctx); GGML_API int gguf_get_version (struct gguf_context * ctx); GGML_API size_t gguf_get_alignment (struct gguf_context * ctx); GGML_API size_t gguf_get_data_offset(struct gguf_context * ctx); + GGML_API void * gguf_get_data (struct gguf_context * ctx); GGML_API int gguf_get_n_kv(struct gguf_context * ctx); GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); From 860c9c63ce204dea31fe994ef370c569f7a75596 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 16:36:03 +0300 Subject: [PATCH 008/242] gguf : add gguf_get_tensor_name() --- ggml.c | 4 ++++ ggml.h | 1 + 2 files changed, 5 insertions(+) diff --git a/ggml.c b/ggml.c index f252363d960a3..0304750623148 100644 --- a/ggml.c +++ b/ggml.c @@ -18770,6 +18770,10 @@ size_t gguf_get_tensor_offset(struct gguf_context * ctx, int i) { return ctx->infos[i].offset; } +char * gguf_get_tensor_name(struct gguf_context * ctx, int i) { + return ctx->infos[i].name.data; +} + //////////////////////////////////////////////////////////////////////////////// int ggml_cpu_has_avx(void) { diff --git a/ggml.h b/ggml.h index fac0f5e687e23..1a748d8d82082 100644 --- a/ggml.h +++ b/ggml.h @@ -1670,6 +1670,7 @@ extern "C" { GGML_API int gguf_get_n_tensors (struct gguf_context * ctx); GGML_API size_t gguf_get_tensor_offset(struct gguf_context * ctx, int i); + GGML_API char * gguf_get_tensor_name (struct gguf_context * ctx, int i); // // system info From cb871fa022aa7a8b72c3a616f7ac7e8e9f1748d9 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 18:48:52 +0300 Subject: [PATCH 009/242] gguf : do not support passing existing ggml_context to gguf_init --- ggml.c | 51 ++++++++++++++++----------------------------------- ggml.h | 5 ++--- 2 files changed, 18 insertions(+), 38 deletions(-) diff --git a/ggml.c b/ggml.c index 0304750623148..1b14c37905b56 100644 --- a/ggml.c +++ b/ggml.c @@ -18388,6 +18388,8 @@ static bool gguf_fread_str(void * dst, FILE * file, size_t * offset) { } struct gguf_context * gguf_init(const char * fname, struct gguf_init_params params) { + GGML_ASSERT(!params.load || params.malloc || params.ctx != NULL); + FILE * file = fopen(fname, "rb"); if (!file) { return NULL; @@ -18518,8 +18520,7 @@ struct gguf_context * gguf_init(const char * fname, struct gguf_init_params para const size_t size_cur = (ne*ggml_type_size(info->type))/ggml_blck_size(info->type); - // TODO: pad size_cur to alignment - ctx->size_data += size_cur; + ctx->size_data += GGML_PAD(size_cur, ctx->alignment); } // TODO: simplify @@ -18528,28 +18529,18 @@ struct gguf_context * gguf_init(const char * fname, struct gguf_init_params para ctx->data = GGML_ALIGNED_MALLOC(ctx->size_data); fseek(file, ctx->offset, SEEK_SET); ok = ok && gguf_fread_el(ctx->data, ctx->size_data, file, &offset); - } else if (params.ctx != NULL) { - bool ctx_new = false; - bool ctx_no_alloc = false; - - if (*params.ctx == NULL) { - const size_t mem_size = - ctx->header.n_tensors*ggml_tensor_overhead() + 1 + - ctx->size_data; - - struct ggml_init_params pdata = { - .mem_size = mem_size, - .mem_buffer = NULL, - .no_alloc = false, - }; + } else { + const size_t mem_size = + ctx->header.n_tensors*ggml_tensor_overhead() + 1 + + ctx->size_data; - *params.ctx = ggml_init(pdata); + struct ggml_init_params pdata = { + .mem_size = mem_size, + .mem_buffer = NULL, + .no_alloc = false, + }; - ctx_new = true; - } else { - ctx_no_alloc = ggml_get_no_alloc(*params.ctx); - ggml_set_no_alloc(*params.ctx, false); - } + *params.ctx = ggml_init(pdata); struct ggml_context * ctx_data = *params.ctx; @@ -18561,11 +18552,7 @@ struct gguf_context * gguf_init(const char * fname, struct gguf_init_params para if (!ok) { fprintf(stderr, "%s: failed to read tensor data\n", __func__); fclose(file); - if (ctx_new) { - ggml_free(ctx_data); - } else { - ggml_set_no_alloc(ctx_data, ctx_no_alloc); - } + ggml_free(ctx_data); gguf_free(ctx); return NULL; } @@ -18597,18 +18584,12 @@ struct gguf_context * gguf_init(const char * fname, struct gguf_init_params para if (!ok) { fprintf(stderr, "%s: failed to create tensors\n", __func__); fclose(file); - if (ctx_new) { - ggml_free(ctx_data); - } else { - ggml_set_no_alloc(ctx_data, ctx_no_alloc); - } + ggml_free(ctx_data); gguf_free(ctx); return NULL; } - ggml_set_no_alloc(ctx_data, ctx_no_alloc); - } else { - GGML_ASSERT("gguf: invalid params - load requires malloc or ctx"); + ggml_set_no_alloc(ctx_data, false); } } diff --git a/ggml.h b/ggml.h index 1a748d8d82082..7d5514ba394f5 100644 --- a/ggml.h +++ b/ggml.h @@ -1636,9 +1636,8 @@ extern "C" { struct gguf_init_params { bool load; // load the tensor data - bool malloc; // if false, use the provided ggml_context to allocate the tensor data - // it no ggml_context is provided, it will be created - // if true, use malloc to allocate the tensor data + bool malloc; // if false, create a ggml_context and allocate the tensor data in it + // if true, use malloc to allocate the tensor data instead struct ggml_context ** ctx; }; From d313c0fa33dc284e23a88fae90b1f94cb0ff6f5c Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 18:53:57 +0300 Subject: [PATCH 010/242] gguf : simplify gguf_get_val --- ggml.c | 68 +++++++++++++++++++--------------------------------------- ggml.h | 20 +++-------------- 2 files changed, 25 insertions(+), 63 deletions(-) diff --git a/ggml.c b/ggml.c index 1b14c37905b56..e68e91e18646b 100644 --- a/ggml.c +++ b/ggml.c @@ -18297,6 +18297,19 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i //////////////////////////////////////////////////////////////////////////////// +enum gguf_type { + GGUF_TYPE_UINT8 = 0, + GGUF_TYPE_INT8 = 1, + GGUF_TYPE_UINT16 = 2, + GGUF_TYPE_INT16 = 3, + GGUF_TYPE_UINT32 = 4, + GGUF_TYPE_INT32 = 5, + GGUF_TYPE_FLOAT32 = 6, + GGUF_TYPE_BOOL = 7, + GGUF_TYPE_STRING = 8, + GGUF_TYPE_ARRAY = 9, +}; + struct gguf_str { uint32_t n; char * data; @@ -18670,77 +18683,40 @@ enum gguf_type gguf_get_type(struct gguf_context * ctx, int i) { return ctx->header.kv[i].type; } -void gguf_get_val(struct gguf_context * ctx, int i, void * val) { - struct gguf_kv * kv = &ctx->header.kv[i]; - - switch (kv->type) { - case GGUF_TYPE_UINT8: memcpy(val, &kv->value.uint8, sizeof(uint8_t)); break; - case GGUF_TYPE_INT8: memcpy(val, &kv->value.int8, sizeof(int8_t)); break; - case GGUF_TYPE_UINT16: memcpy(val, &kv->value.uint16, sizeof(uint16_t)); break; - case GGUF_TYPE_INT16: memcpy(val, &kv->value.int16, sizeof(int16_t)); break; - case GGUF_TYPE_UINT32: memcpy(val, &kv->value.uint32, sizeof(uint32_t)); break; - case GGUF_TYPE_INT32: memcpy(val, &kv->value.int32, sizeof(int32_t)); break; - case GGUF_TYPE_FLOAT32: memcpy(val, &kv->value.float32, sizeof(float)); break; - case GGUF_TYPE_BOOL: memcpy(val, &kv->value.bool_, sizeof(bool)); break; - case GGUF_TYPE_STRING: memcpy(val, &kv->value.str.data, sizeof(char *)); break; - default: - GGML_ASSERT("gguf: not implemented"); - break; - } -} - uint8_t gguf_get_val_u8(struct gguf_context * ctx, int i) { - uint8_t val; - gguf_get_val(ctx, i, &val); - return val; + return ctx->header.kv[i].value.uint8; } int8_t gguf_get_val_i8(struct gguf_context * ctx, int i) { - int8_t val; - gguf_get_val(ctx, i, &val); - return val; + return ctx->header.kv[i].value.int8; } uint16_t gguf_get_val_u16(struct gguf_context * ctx, int i) { - uint16_t val; - gguf_get_val(ctx, i, &val); - return val; + return ctx->header.kv[i].value.uint16; } int16_t gguf_get_val_i16(struct gguf_context * ctx, int i) { - int16_t val; - gguf_get_val(ctx, i, &val); - return val; + return ctx->header.kv[i].value.int16; } uint32_t gguf_get_val_u32(struct gguf_context * ctx, int i) { - uint32_t val; - gguf_get_val(ctx, i, &val); - return val; + return ctx->header.kv[i].value.uint32; } int32_t gguf_get_val_i32(struct gguf_context * ctx, int i) { - int32_t val; - gguf_get_val(ctx, i, &val); - return val; + return ctx->header.kv[i].value.int32; } float gguf_get_val_f32(struct gguf_context * ctx, int i) { - float val; - gguf_get_val(ctx, i, &val); - return val; + return ctx->header.kv[i].value.float32; } bool gguf_get_val_bool(struct gguf_context * ctx, int i) { - bool val; - gguf_get_val(ctx, i, &val); - return val; + return ctx->header.kv[i].value.bool_; } const char * gguf_get_val_str (struct gguf_context * ctx, int i) { - char * val; - gguf_get_val(ctx, i, &val); - return val; + return ctx->header.kv[i].value.str.data; } int gguf_get_n_tensors(struct gguf_context * ctx) { diff --git a/ggml.h b/ggml.h index 7d5514ba394f5..75a41a28f6653 100644 --- a/ggml.h +++ b/ggml.h @@ -1619,19 +1619,6 @@ extern "C" { // gguf // - enum gguf_type { - GGUF_TYPE_UINT8 = 0, - GGUF_TYPE_INT8 = 1, - GGUF_TYPE_UINT16 = 2, - GGUF_TYPE_INT16 = 3, - GGUF_TYPE_UINT32 = 4, - GGUF_TYPE_INT32 = 5, - GGUF_TYPE_FLOAT32 = 6, - GGUF_TYPE_BOOL = 7, - GGUF_TYPE_STRING = 8, - GGUF_TYPE_ARRAY = 9, - }; - struct gguf_context; struct gguf_init_params { @@ -1651,10 +1638,9 @@ extern "C" { GGML_API size_t gguf_get_data_offset(struct gguf_context * ctx); GGML_API void * gguf_get_data (struct gguf_context * ctx); - GGML_API int gguf_get_n_kv(struct gguf_context * ctx); - GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); - GGML_API enum gguf_type gguf_get_type(struct gguf_context * ctx, int i); - GGML_API void gguf_get_val (struct gguf_context * ctx, int i, void * val); + GGML_API int gguf_get_n_kv(struct gguf_context * ctx); + GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); + GGML_API void gguf_get_val (struct gguf_context * ctx, int i, void * val); GGML_API uint8_t gguf_get_val_u8 (struct gguf_context * ctx, int i); GGML_API int8_t gguf_get_val_i8 (struct gguf_context * ctx, int i); From e46870f5af3432212982645fcd1cc59b8e106734 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 18:55:32 +0300 Subject: [PATCH 011/242] gguf : gguf.c is now part of ggml.c --- gguf.c | 192 --------------------------------------------------------- 1 file changed, 192 deletions(-) delete mode 100644 gguf.c diff --git a/gguf.c b/gguf.c deleted file mode 100644 index 54b31d411f35b..0000000000000 --- a/gguf.c +++ /dev/null @@ -1,192 +0,0 @@ -// TODO: convert to proper gguf.h gguf.c structure, now I'm trying to be fast as much as possible, -// and everything is in this file for quick debugging. - -#include -#include -#include -#include - - -enum ggml_type { - GGML_TYPE_F32 = 0, - GGML_TYPE_F16 = 1, - GGML_TYPE_Q4_0 = 2, - GGML_TYPE_Q4_1 = 3, - // GGML_TYPE_Q4_2 = 4, support has been removed - // GGML_TYPE_Q4_3 (5) support has been removed - GGML_TYPE_Q5_0 = 6, - GGML_TYPE_Q5_1 = 7, - GGML_TYPE_Q8_0 = 8, - GGML_TYPE_Q8_1 = 9, - // k-quantizations - GGML_TYPE_Q2_K = 10, - GGML_TYPE_Q3_K = 11, - GGML_TYPE_Q4_K = 12, - GGML_TYPE_Q5_K = 13, - GGML_TYPE_Q6_K = 14, - GGML_TYPE_Q8_K = 15, - GGML_TYPE_I8, - GGML_TYPE_I16, - GGML_TYPE_I32, - GGML_TYPE_COUNT, -}; - -enum gguf_metadata_value_type { - GGUF_METADATA_VALUE_TYPE_UINT8 = 0, - GGUF_METADATA_VALUE_TYPE_INT8 = 1, - GGUF_METADATA_VALUE_TYPE_UINT16 = 2, - GGUF_METADATA_VALUE_TYPE_INT16 = 3, - GGUF_METADATA_VALUE_TYPE_UINT32 = 4, - GGUF_METADATA_VALUE_TYPE_INT32 = 5, - GGUF_METADATA_VALUE_TYPE_FLOAT32 = 6, - GGUF_METADATA_VALUE_TYPE_BOOL = 7, - GGUF_METADATA_VALUE_TYPE_STRING = 8, - GGUF_METADATA_VALUE_TYPE_ARRAY = 9, -}; - -struct gguf_string_t { - uint32_t len; - char * string; -}; - -union gguf_metadata_value_t; - -// Union definition for gguf_metadata_value_t -union gguf_metadata_value_t { - uint8_t uint8; - int8_t int8; - uint16_t uint16; - int16_t int16; - uint32_t uint32; - int32_t int32; - float float32; - bool bool_; - struct gguf_string_t string; - struct { - uint32_t len; - enum gguf_metadata_value_type type; - union gguf_metadata_value_t * array; - } array; -}; - - -struct gguf_metadata_kv_t { - struct gguf_string_t key; - uint32_t value_len; - enum gguf_metadata_value_type value_type; - union gguf_metadata_value_t* value; -}; - -struct gguf_header_t { - uint32_t magic; - uint32_t version; - uint32_t tensor_count; - uint32_t metadata_kv_count; - struct gguf_metadata_kv_t * metadata_kv; -}; - -struct gguf_tensor_info_t { - struct gguf_string_t name; - uint32_t n_dimensions; - uint32_t dimensions[]; -}; - -struct gguf_file_t { - struct gguf_header_t header; - uint8_t tensor_data[]; -}; - -void read_gguf_file(const char * file_path, struct gguf_file_t * gguf_file) { - FILE* file = fopen(file_path, "rb"); - if (file == NULL) { - printf("Error opening the file.\n"); - return; - } - - fread(&gguf_file->header.magic, sizeof(uint32_t), 1, file); - - // Verify magic and version - if (gguf_file->header.magic != 0x47475546) { - printf("Invalid magic number. Not a valid GGUF file.\n"); - fclose(file); - return; - } - - fread(&gguf_file->header.version, sizeof(uint32_t), 1, file); - - if (gguf_file->header.version != 1) { - printf("Unsupported version. Expected version 1.\n"); - fclose(file); - return; - } - - fread(&gguf_file->header.tensor_count, sizeof(uint32_t), 1, file); - fread(&gguf_file->header.metadata_kv_count, sizeof(uint32_t), 1, file); - - printf("Magic: %x\n", gguf_file->header.magic); - printf("Version: %d\n", gguf_file->header.version); - printf("Tensor Count: %d\n", gguf_file->header.tensor_count); - printf("Metadata Key-Value Count: %d\n", gguf_file->header.metadata_kv_count); - - gguf_file->header.metadata_kv = (struct gguf_metadata_kv_t*)malloc(gguf_file->header.metadata_kv_count * sizeof(struct gguf_metadata_kv_t)); - - for (int i = 0; i < gguf_file->header.metadata_kv_count; i++) { - struct gguf_metadata_kv_t* kv = &gguf_file->header.metadata_kv[i]; - fread(&kv->key.len, sizeof(uint32_t), 1, file); - kv->key.string = (char*)malloc(kv->key.len ); // Allocate memory for the key string - fread(kv->key.string, sizeof(char), kv->key.len, file); - //kv->key.string[kv->key.len] = '\0'; // Null-terminate the key string - - fread(&kv->value_type, sizeof(uint32_t), 1, file); - - printf("Metadata Value Type: %d\n", kv->value_type); - printf("Metadata Key: %s\n", kv->key.string); - - // Read metadata value according to its type using reinterpret_cast - switch (kv->value_type) { - case GGUF_METADATA_VALUE_TYPE_UINT32: - kv->value = (uint32_t *) malloc(sizeof(uint32_t)); - fread(kv->value, sizeof(uint32_t), 1, file); - printf("value: %d\n", kv->value->uint32); - break; - case GGUF_METADATA_VALUE_TYPE_FLOAT32: - kv->value = (float *)malloc(sizeof(float)); - fread(kv->value, sizeof(float), 1, file); - printf("value: %f\n", (float)kv->value->float32); - break; - case GGUF_METADATA_VALUE_TYPE_STRING: - fread(&kv->value_len, sizeof(uint32_t), 1, file); - printf("value len: %d\n", kv->value_len); -kv->value = (char *)malloc(sizeof(char) * kv->value_len); // Allocate memory for the value string -fread(kv->value, sizeof(char), kv->value_len, file); - printf("value: %s\n", (char *)kv->value); - break; - // ... (handle other types in a similar manner) - default: - printf("Unsupported metadata value type.\n"); - fclose(file); - return; - } - } - - // TODO: handle reading tensor data - - fclose(file); -} - -void gguf_free(struct gguf_file_t * gguf_file) { - // Free allocated memory for key strings avd values - for (int i = 0; i < gguf_file->header.metadata_kv_count; i++) { - free(gguf_file->header.metadata_kv[i].key.string); - free(gguf_file->header.metadata_kv[i].value); - } - free(gguf_file->header.metadata_kv); -} - -int main() { - const char* file_path = "example.gguf"; - struct gguf_file_t gguf_file; - read_gguf_file(file_path, &gguf_file); - gguf_free(&gguf_file); - return 0; -} From 5628ec71636e0b390213caa4c273d3ef8bbd7459 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 20:04:22 +0300 Subject: [PATCH 012/242] gguf : read / write sample models --- examples/gguf/gguf.cpp | 323 ++++++++++++++++++++++++++++++++++++++++- ggml.c | 117 ++++++++------- ggml.h | 5 +- 3 files changed, 385 insertions(+), 60 deletions(-) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index 602de519adb1e..a5c442ac581f5 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -1,15 +1,326 @@ #include "ggml.h" #include +#include #include +#include +#include +#include -bool gguf_write(const std::string & fname) { +enum gguf_type { + GGUF_TYPE_UINT8 = 0, + GGUF_TYPE_INT8 = 1, + GGUF_TYPE_UINT16 = 2, + GGUF_TYPE_INT16 = 3, + GGUF_TYPE_UINT32 = 4, + GGUF_TYPE_INT32 = 5, + GGUF_TYPE_FLOAT32 = 6, + GGUF_TYPE_BOOL = 7, + GGUF_TYPE_STRING = 8, + GGUF_TYPE_ARRAY = 9, +}; +template +static std::string to_string(const T & val) { + std::stringstream ss; + ss << val; + return ss.str(); +} + +void gguf_ex_write_str(std::ofstream & fout, const std::string & val) { + const int32_t n = val.size(); + fout.write((const char *) &n, sizeof(n)); + fout.write(val.c_str(), n); +} + +void gguf_ex_write_i32(std::ofstream & fout, int32_t val) { + fout.write((const char *) &val, sizeof(val)); +} + +void gguf_ex_write_u64(std::ofstream & fout, size_t val) { + fout.write((const char *) &val, sizeof(val)); +} + +template +void gguf_ex_write_param(std::ofstream & fout, const std::string & key, enum gguf_type type, const T & val) { + gguf_ex_write_str(fout, key); + fout.write((const char *) &type, sizeof(type)); + fout.write((const char *) &val, sizeof(val)); + + fprintf(stdout, "%s: write param: %s = %s\n", __func__, key.c_str(), to_string(val).c_str()); +} + +template<> +void gguf_ex_write_param(std::ofstream & fout, const std::string & key, enum gguf_type type, const std::string & val) { + gguf_ex_write_str(fout, key); + fout.write((const char *) &type, sizeof(type)); + + const int32_t n = val.size(); + fout.write((const char *) &n, sizeof(n)); + fout.write(val.c_str(), n); +} + +bool gguf_ex_write(const std::string & fname) { + std::ofstream fout(fname.c_str(), std::ios::binary); + + { + const int32_t magic = GGUF_MAGIC; + fout.write((const char *) &magic, sizeof(magic)); + } + + { + const int32_t version = GGUF_VERSION; + fout.write((const char *) &version, sizeof(version)); + } + + const int n_tensors = 10; + const int n_kv = 9; + + fout.write((const char*) &n_tensors, sizeof(n_tensors)); + fout.write((const char*) &n_kv, sizeof(n_kv)); + + fprintf(stdout, "%s: write header\n", __func__); + + // kv data + { + gguf_ex_write_param< uint8_t>(fout, "some.parameter.uint8", GGUF_TYPE_UINT8, 0x12); + gguf_ex_write_param< int8_t>(fout, "some.parameter.int8", GGUF_TYPE_INT8, -0x13); + gguf_ex_write_param(fout, "some.parameter.uint16", GGUF_TYPE_UINT16, 0x1234); + gguf_ex_write_param< int16_t>(fout, "some.parameter.int16", GGUF_TYPE_INT16, -0x1235); + gguf_ex_write_param(fout, "some.parameter.uint32", GGUF_TYPE_UINT32, 0x12345678); + gguf_ex_write_param< int32_t>(fout, "some.parameter.int32", GGUF_TYPE_INT32, -0x12345679); + + gguf_ex_write_param (fout, "some.parameter.float32", GGUF_TYPE_FLOAT32, 0.123456789f); + gguf_ex_write_param (fout, "some.parameter.bool", GGUF_TYPE_BOOL, true); + + gguf_ex_write_param(fout, "some.parameter.string", GGUF_TYPE_STRING, "hello world"); + } + + uint64_t offset_tensor = 0; + + struct ggml_init_params params = { + /*.mem_size =*/ 128ull*1024ull*1024ull, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + + struct ggml_context * ctx_data = ggml_init(params); + + // tensor infos + for (int i = 0; i < n_tensors; ++i) { + const std::string name = "tensor_" + to_string(i); + + int64_t ne[GGML_MAX_DIMS] = { 1 }; + int32_t n_dims = rand() % GGML_MAX_DIMS + 1; + + for (int j = 0; j < n_dims; ++j) { + ne[j] = rand() % 10 + 1; + } + + struct ggml_tensor * cur = ggml_new_tensor(ctx_data, GGML_TYPE_F32, n_dims, ne); + ggml_set_name(cur, name.c_str()); + + { + float * data = (float *) cur->data; + for (int j = 0; j < ggml_nelements(cur); ++j) { + data[j] = 100 + i; + } + } + + fprintf(stdout, "%s: tensor: %s, %d dims, ne = [", __func__, name.c_str(), n_dims); + for (int j = 0; j < 4; ++j) { + fprintf(stdout, "%s%3d", j == 0 ? "" : ", ", (int) cur->ne[j]); + } + fprintf(stdout, "], offset_tensor = %6" PRIu64 "\n", offset_tensor); + + gguf_ex_write_str(fout, name); + gguf_ex_write_i32(fout, n_dims); + for (int j = 0; j < n_dims; ++j) { + gguf_ex_write_i32(fout, cur->ne[j]); + } + gguf_ex_write_i32(fout, cur->type); + gguf_ex_write_u64(fout, offset_tensor); + + offset_tensor += GGML_PAD(ggml_nbytes(cur), GGUF_DEFAULT_ALIGNMENT); + } + + const uint64_t offset_data = GGML_PAD((uint64_t) fout.tellp(), GGUF_DEFAULT_ALIGNMENT); + + fprintf(stdout, "%s: data offset = %" PRIu64 "\n", __func__, offset_data); + + { + const size_t pad = offset_data - fout.tellp(); + + for (size_t j = 0; j < pad; ++j) { + fout.put(0); + } + } + + for (int i = 0; i < n_tensors; ++i) { + fprintf(stdout, "%s: writing tensor %d data\n", __func__, i); + + const std::string name = "tensor_" + to_string(i); + + struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name.c_str()); + + fout.write((const char *) cur->data, ggml_nbytes(cur)); + + { + const size_t pad = GGML_PAD(ggml_nbytes(cur), GGUF_DEFAULT_ALIGNMENT) - ggml_nbytes(cur); + + for (size_t j = 0; j < pad; ++j) { + fout.put(0); + } + } + } + + fout.close(); + + fprintf(stdout, "%s: wrote file '%s;\n", __func__, fname.c_str()); + + ggml_free(ctx_data); + + return true; +} + +// just read tensor info +bool gguf_ex_read_0(const std::string & fname) { + struct gguf_init_params params = { + /*.no_alloc = */ false, + /*.ctx = */ NULL, + }; + + struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params); + + fprintf(stdout, "%s: version: %d\n", __func__, gguf_get_version(ctx)); + fprintf(stdout, "%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); + fprintf(stdout, "%s: data offset: %zu\n", __func__, gguf_get_data_offset(ctx)); + + // kv + { + const int n_kv = gguf_get_n_kv(ctx); + + fprintf(stdout, "%s: n_kv: %d\n", __func__, n_kv); + + for (int i = 0; i < n_kv; ++i) { + const char * key = gguf_get_key(ctx, i); + + fprintf(stdout, "%s: kv[%d]: key = %s\n", __func__, i, key); + } + } + + // tensor info + { + const int n_tensors = gguf_get_n_tensors(ctx); + + fprintf(stdout, "%s: n_tensors: %d\n", __func__, n_tensors); + + for (int i = 0; i < n_tensors; ++i) { + const char * name = gguf_get_tensor_name(ctx, i); + const size_t offset = gguf_get_tensor_offset(ctx, i); + + fprintf(stdout, "%s: tensor[%d]: name = %s, offset = %zu\n", __func__, i, name, offset); + } + } + + return true; +} + +// read and create ggml_context containing the tensors and their data +bool gguf_ex_read_1(const std::string & fname) { + struct ggml_context * ctx_data = NULL; + + struct gguf_init_params params = { + /*.no_alloc = */ false, + /*.ctx = */ &ctx_data, + }; + + struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params); + + fprintf(stdout, "%s: version: %d\n", __func__, gguf_get_version(ctx)); + fprintf(stdout, "%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); + fprintf(stdout, "%s: data offset: %zu\n", __func__, gguf_get_data_offset(ctx)); + + // kv + { + const int n_kv = gguf_get_n_kv(ctx); + + fprintf(stdout, "%s: n_kv: %d\n", __func__, n_kv); + + for (int i = 0; i < n_kv; ++i) { + const char * key = gguf_get_key(ctx, i); + + fprintf(stdout, "%s: kv[%d]: key = %s\n", __func__, i, key); + } + } + + // tensor info + { + const int n_tensors = gguf_get_n_tensors(ctx); + + fprintf(stdout, "%s: n_tensors: %d\n", __func__, n_tensors); + + for (int i = 0; i < n_tensors; ++i) { + const char * name = gguf_get_tensor_name(ctx, i); + const size_t offset = gguf_get_tensor_offset(ctx, i); + + fprintf(stdout, "%s: tensor[%d]: name = %s, offset = %zu\n", __func__, i, name, offset); + } + } + + // data + { + const int n_tensors = gguf_get_n_tensors(ctx); + + for (int i = 0; i < n_tensors; ++i) { + fprintf(stdout, "%s: reading tensor %d data\n", __func__, i); + + const std::string name = "tensor_" + to_string(i); + + struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name.c_str()); + + fprintf(stdout, "%s: tensor[%d]: n_dims = %d, name = %s, data = %p\n", + __func__, i, cur->n_dims, cur->name, cur->data); + + // check data + { + const float * data = (const float *) cur->data; + for (int j = 0; j < ggml_nelements(cur); ++j) { + if (data[j] != 100 + i) { + fprintf(stderr, "%s: tensor[%d]: data[%d] = %f\n", __func__, i, j, data[j]); + return false; + } + } + } + } + } + + fprintf(stdout, "%s: ctx_data size: %zu\n", __func__, ggml_get_mem_size(ctx_data)); + + ggml_free(ctx_data); + gguf_free(ctx); return true; } -bool gguf_read(const std::string & fname) { +// read just the tensor info and mmap the data in user code +bool gguf_ex_read_2(const std::string & fname) { + struct ggml_context * ctx_data = NULL; + + struct gguf_init_params params = { + /*.no_alloc = */ true, + /*.ctx = */ &ctx_data, + }; + + struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params); + + // TODO: mmap based on tensor infos + + fprintf(stdout, "%s: ctx_data size: %zu\n", __func__, ggml_get_mem_size(ctx_data)); + + ggml_free(ctx_data); + gguf_free(ctx); + return true; } @@ -20,14 +331,16 @@ int main(int argc, char ** argv) { } const std::string fname(argv[1]); - const std::string mode(argv[2]); + const std::string mode (argv[2]); GGML_ASSERT((mode == "r" || mode == "w") && "mode must be r or w"); if (mode == "w") { - GGML_ASSERT(gguf_write(fname) && "failed to write gguf file"); + GGML_ASSERT(gguf_ex_write(fname) && "failed to write gguf file"); } else if (mode == "r") { - GGML_ASSERT(gguf_read(fname) && "failed to read gguf file"); + GGML_ASSERT(gguf_ex_read_0(fname) && "failed to read gguf file"); + GGML_ASSERT(gguf_ex_read_1(fname) && "failed to read gguf file"); + GGML_ASSERT(gguf_ex_read_2(fname) && "failed to read gguf file"); } return 0; diff --git a/ggml.c b/ggml.c index e68e91e18646b..5736c800ebe69 100644 --- a/ggml.c +++ b/ggml.c @@ -18364,7 +18364,7 @@ struct gguf_tensor_info { enum ggml_type type; - uint64_t offset; // offset from beginning of file, must be a multiple of `ALIGNMENT` + uint64_t offset; // offset from start of `data`, must be a multiple of `ALIGNMENT` }; struct gguf_context { @@ -18385,9 +18385,7 @@ static bool gguf_fread_el(void * dst, size_t size, FILE * file, size_t * offset) return n == size; } -static bool gguf_fread_str(void * dst, FILE * file, size_t * offset) { - struct gguf_str * p = (struct gguf_str *) dst; - +static bool gguf_fread_str(struct gguf_str * p, FILE * file, size_t * offset) { p->n = 0; p->data = NULL; @@ -18395,14 +18393,12 @@ static bool gguf_fread_str(void * dst, FILE * file, size_t * offset) { // TODO: how to avoid mallocs for strings? ok = ok && gguf_fread_el(&p->n, sizeof(p->n), file, offset); p->data = calloc(p->n + 1, 1); - ok = ok && gguf_fread_el(&p->data, p->n, file, offset); + ok = ok && gguf_fread_el( p->data, p->n, file, offset); return ok; } -struct gguf_context * gguf_init(const char * fname, struct gguf_init_params params) { - GGML_ASSERT(!params.load || params.malloc || params.ctx != NULL); - +struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) { FILE * file = fopen(fname, "rb"); if (!file) { return NULL; @@ -18446,10 +18442,14 @@ struct gguf_context * gguf_init(const char * fname, struct gguf_init_params para for (uint32_t i = 0; i < ctx->header.n_kv; ++i) { struct gguf_kv * kv = &ctx->header.kv[i]; + //fprintf(stderr, "%s: reading kv %d\n", __func__, i); + ok = ok && gguf_fread_str(&kv->key, file, &offset); //ok = ok && gguf_fread_el (&kv->n_bytes, sizeof(kv->n_bytes), file, &offset); ok = ok && gguf_fread_el (&kv->type, sizeof(kv->type), file, &offset); + //fprintf(stderr, "%s: reading kv with key %s\n", __func__, kv->key.data); + switch (kv->type) { case GGUF_TYPE_UINT8: ok = ok && gguf_fread_el (&kv->value.uint8, sizeof(kv->value.uint8), file, &offset); break; case GGUF_TYPE_INT8: ok = ok && gguf_fread_el (&kv->value.int8, sizeof(kv->value.int8), file, &offset); break; @@ -18461,9 +18461,13 @@ struct gguf_context * gguf_init(const char * fname, struct gguf_init_params para case GGUF_TYPE_BOOL: ok = ok && gguf_fread_el (&kv->value.bool_, sizeof(kv->value.bool_), file, &offset); break; case GGUF_TYPE_STRING: ok = ok && gguf_fread_str(&kv->value.str, file, &offset); break; case GGUF_TYPE_ARRAY: - GGML_ASSERT("gguf: array type not implemented"); - break; - }; + GGML_ASSERT("gguf: array type not implemented"); + break; + }; + + if (!ok) { + break; + } } if (!ok) { @@ -18478,12 +18482,14 @@ struct gguf_context * gguf_init(const char * fname, struct gguf_init_params para for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { struct gguf_tensor_info * info = &ctx->infos[i]; - memset(info->ne, 1, sizeof(info->ne)); + for (int j = 0; j < GGML_MAX_DIMS; ++j) { + info->ne[j] = 1; + } ok = ok && gguf_fread_str(&info->name, file, &offset); ok = ok && gguf_fread_el (&info->n_dims, sizeof(info->n_dims), file, &offset); for (uint32_t j = 0; j < info->n_dims; ++j) { - ok = ok && gguf_fread_el (&info->ne[j], sizeof(info->ne[j]), file, &offset); + ok = ok && gguf_fread_el(&info->ne[j], sizeof(info->ne[j]), file, &offset); } //ok = ok && gguf_fread_el (&info->n_elms, sizeof(info->n_elms), file, &offset); ok = ok && gguf_fread_el (&info->type, sizeof(info->type), file, &offset); @@ -18536,28 +18542,30 @@ struct gguf_context * gguf_init(const char * fname, struct gguf_init_params para ctx->size_data += GGML_PAD(size_cur, ctx->alignment); } + // load the tensor data // TODO: simplify - if (params.load) { - if (params.malloc) { - ctx->data = GGML_ALIGNED_MALLOC(ctx->size_data); - fseek(file, ctx->offset, SEEK_SET); - ok = ok && gguf_fread_el(ctx->data, ctx->size_data, file, &offset); - } else { - const size_t mem_size = - ctx->header.n_tensors*ggml_tensor_overhead() + 1 + - ctx->size_data; + if (params.ctx != NULL) { + const size_t mem_size = + params.no_alloc ? + (ctx->header.n_tensors + 1)*ggml_tensor_overhead() : + (ctx->header.n_tensors + 1)*ggml_tensor_overhead() + ctx->size_data; + + struct ggml_init_params pdata = { + .mem_size = mem_size, + .mem_buffer = NULL, + .no_alloc = params.no_alloc, + }; - struct ggml_init_params pdata = { - .mem_size = mem_size, - .mem_buffer = NULL, - .no_alloc = false, - }; + *params.ctx = ggml_init(pdata); + + struct ggml_context * ctx_data = *params.ctx; - *params.ctx = ggml_init(pdata); + struct ggml_tensor * data = NULL; - struct ggml_context * ctx_data = *params.ctx; + if (params.no_alloc == false) { + data = ggml_new_tensor_1d(ctx_data, GGML_TYPE_I8, ctx->size_data); - struct ggml_tensor * data = ggml_new_tensor_1d(ctx_data, GGML_TYPE_I8, ctx->size_data); + ok = ok && data != NULL; // read the tensor data ok = ok && gguf_fread_el(data->data, ctx->size_data, file, &offset); @@ -18571,39 +18579,44 @@ struct gguf_context * gguf_init(const char * fname, struct gguf_init_params para } ctx->data = data->data; + } - // create the tensors - ggml_set_no_alloc(ctx_data, true); + ggml_set_no_alloc(ctx_data, true); - for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { - const int64_t ne[GGML_MAX_DIMS] = { - ctx->infos[i].ne[0], - ctx->infos[i].ne[1], - ctx->infos[i].ne[2], - ctx->infos[i].ne[3], - }; + // create the tensors + for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { + const int64_t ne[GGML_MAX_DIMS] = { + ctx->infos[i].ne[0], + ctx->infos[i].ne[1], + ctx->infos[i].ne[2], + ctx->infos[i].ne[3], + }; - struct ggml_tensor * cur = ggml_new_tensor(ctx_data, ctx->infos[i].type, ctx->infos[i].n_dims, ne); + struct ggml_tensor * cur = ggml_new_tensor(ctx_data, ctx->infos[i].type, ctx->infos[i].n_dims, ne); - ok = ok && cur != NULL; + ok = ok && cur != NULL; - if (!ok) { - break; - } + ggml_set_name(cur, ctx->infos[i].name.data); - cur->data = (char *) data->data + ctx->infos[i].offset - ctx->offset; + if (!ok) { + break; } - if (!ok) { - fprintf(stderr, "%s: failed to create tensors\n", __func__); - fclose(file); - ggml_free(ctx_data); - gguf_free(ctx); - return NULL; + if (params.no_alloc == false) { + //cur->data = (char *) data->data + ctx->infos[i].offset - ctx->offset; // offset from start of file + cur->data = (char *) data->data + ctx->infos[i].offset; // offset from data } + } - ggml_set_no_alloc(ctx_data, false); + if (!ok) { + fprintf(stderr, "%s: failed to create tensors\n", __func__); + fclose(file); + ggml_free(ctx_data); + gguf_free(ctx); + return NULL; } + + ggml_set_no_alloc(ctx_data, params.no_alloc); } if (!ok) { diff --git a/ggml.h b/ggml.h index 75a41a28f6653..e0abbbfdd81e3 100644 --- a/ggml.h +++ b/ggml.h @@ -1622,10 +1622,9 @@ extern "C" { struct gguf_context; struct gguf_init_params { - bool load; // load the tensor data - bool malloc; // if false, create a ggml_context and allocate the tensor data in it - // if true, use malloc to allocate the tensor data instead + bool no_alloc; + // if not NULL, create a ggml_context and allocate the tensor data in it struct ggml_context ** ctx; }; From d8491fc7e3545a4447555cb9e4487994bc98c024 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 26 Jul 2023 22:56:26 +0300 Subject: [PATCH 013/242] gguf : add comments --- ggml.c | 212 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 114 insertions(+), 98 deletions(-) diff --git a/ggml.c b/ggml.c index 5736c800ebe69..b005fd8893e58 100644 --- a/ggml.c +++ b/ggml.c @@ -18407,99 +18407,112 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p // offset from start of file size_t offset = 0; - // check the magic before making allocations uint32_t magic = 0; - gguf_fread_el(&magic, sizeof(magic), file, &offset); - if (magic != GGUF_MAGIC) { - fprintf(stderr, "%s: invalid magic number %08x\n", __func__, magic); - fclose(file); - return NULL; + + // check the magic before making allocations + { + gguf_fread_el(&magic, sizeof(magic), file, &offset); + + if (magic != GGUF_MAGIC) { + fprintf(stderr, "%s: invalid magic number %08x\n", __func__, magic); + fclose(file); + return NULL; + } } bool ok = true; struct gguf_context * ctx = GGML_ALIGNED_MALLOC(sizeof(struct gguf_context)); - ctx->header.magic = magic; - ctx->header.kv = NULL; + // read the header + { + ctx->header.magic = magic; + ctx->header.kv = NULL; - ctx->infos = NULL; - ctx->data = NULL; + ctx->infos = NULL; + ctx->data = NULL; - ok = ok && gguf_fread_el(&ctx->header.version, sizeof(ctx->header.version), file, &offset); - ok = ok && gguf_fread_el(&ctx->header.n_tensors, sizeof(ctx->header.n_tensors), file, &offset); - ok = ok && gguf_fread_el(&ctx->header.n_kv, sizeof(ctx->header.n_kv), file, &offset); + ok = ok && gguf_fread_el(&ctx->header.version, sizeof(ctx->header.version), file, &offset); + ok = ok && gguf_fread_el(&ctx->header.n_tensors, sizeof(ctx->header.n_tensors), file, &offset); + ok = ok && gguf_fread_el(&ctx->header.n_kv, sizeof(ctx->header.n_kv), file, &offset); - if (!ok) { - fprintf(stderr, "%s: failed to read header\n", __func__); - fclose(file); - gguf_free(ctx); - return NULL; + if (!ok) { + fprintf(stderr, "%s: failed to read header\n", __func__); + fclose(file); + gguf_free(ctx); + return NULL; + } } - ctx->header.kv = GGML_ALIGNED_MALLOC(ctx->header.n_kv * sizeof(struct gguf_kv)); - - for (uint32_t i = 0; i < ctx->header.n_kv; ++i) { - struct gguf_kv * kv = &ctx->header.kv[i]; - - //fprintf(stderr, "%s: reading kv %d\n", __func__, i); + // read the kv pairs + { + ctx->header.kv = GGML_ALIGNED_MALLOC(ctx->header.n_kv * sizeof(struct gguf_kv)); - ok = ok && gguf_fread_str(&kv->key, file, &offset); - //ok = ok && gguf_fread_el (&kv->n_bytes, sizeof(kv->n_bytes), file, &offset); - ok = ok && gguf_fread_el (&kv->type, sizeof(kv->type), file, &offset); + for (uint32_t i = 0; i < ctx->header.n_kv; ++i) { + struct gguf_kv * kv = &ctx->header.kv[i]; - //fprintf(stderr, "%s: reading kv with key %s\n", __func__, kv->key.data); + //fprintf(stderr, "%s: reading kv %d\n", __func__, i); + + ok = ok && gguf_fread_str(&kv->key, file, &offset); + //ok = ok && gguf_fread_el (&kv->n_bytes, sizeof(kv->n_bytes), file, &offset); + ok = ok && gguf_fread_el (&kv->type, sizeof(kv->type), file, &offset); + + //fprintf(stderr, "%s: reading kv with key %s\n", __func__, kv->key.data); + + switch (kv->type) { + case GGUF_TYPE_UINT8: ok = ok && gguf_fread_el (&kv->value.uint8, sizeof(kv->value.uint8), file, &offset); break; + case GGUF_TYPE_INT8: ok = ok && gguf_fread_el (&kv->value.int8, sizeof(kv->value.int8), file, &offset); break; + case GGUF_TYPE_UINT16: ok = ok && gguf_fread_el (&kv->value.uint16, sizeof(kv->value.uint16), file, &offset); break; + case GGUF_TYPE_INT16: ok = ok && gguf_fread_el (&kv->value.int16, sizeof(kv->value.int16), file, &offset); break; + case GGUF_TYPE_UINT32: ok = ok && gguf_fread_el (&kv->value.uint32, sizeof(kv->value.uint32), file, &offset); break; + case GGUF_TYPE_INT32: ok = ok && gguf_fread_el (&kv->value.int32, sizeof(kv->value.int32), file, &offset); break; + case GGUF_TYPE_FLOAT32: ok = ok && gguf_fread_el (&kv->value.float32, sizeof(kv->value.float32), file, &offset); break; + case GGUF_TYPE_BOOL: ok = ok && gguf_fread_el (&kv->value.bool_, sizeof(kv->value.bool_), file, &offset); break; + case GGUF_TYPE_STRING: ok = ok && gguf_fread_str(&kv->value.str, file, &offset); break; + case GGUF_TYPE_ARRAY: + GGML_ASSERT("gguf: array type not implemented"); + break; + }; - switch (kv->type) { - case GGUF_TYPE_UINT8: ok = ok && gguf_fread_el (&kv->value.uint8, sizeof(kv->value.uint8), file, &offset); break; - case GGUF_TYPE_INT8: ok = ok && gguf_fread_el (&kv->value.int8, sizeof(kv->value.int8), file, &offset); break; - case GGUF_TYPE_UINT16: ok = ok && gguf_fread_el (&kv->value.uint16, sizeof(kv->value.uint16), file, &offset); break; - case GGUF_TYPE_INT16: ok = ok && gguf_fread_el (&kv->value.int16, sizeof(kv->value.int16), file, &offset); break; - case GGUF_TYPE_UINT32: ok = ok && gguf_fread_el (&kv->value.uint32, sizeof(kv->value.uint32), file, &offset); break; - case GGUF_TYPE_INT32: ok = ok && gguf_fread_el (&kv->value.int32, sizeof(kv->value.int32), file, &offset); break; - case GGUF_TYPE_FLOAT32: ok = ok && gguf_fread_el (&kv->value.float32, sizeof(kv->value.float32), file, &offset); break; - case GGUF_TYPE_BOOL: ok = ok && gguf_fread_el (&kv->value.bool_, sizeof(kv->value.bool_), file, &offset); break; - case GGUF_TYPE_STRING: ok = ok && gguf_fread_str(&kv->value.str, file, &offset); break; - case GGUF_TYPE_ARRAY: - GGML_ASSERT("gguf: array type not implemented"); - break; - }; + if (!ok) { + break; + } + } if (!ok) { - break; + fprintf(stderr, "%s: failed to read key-value pairs\n", __func__); + fclose(file); + gguf_free(ctx); + return NULL; } } - if (!ok) { - fprintf(stderr, "%s: failed to read key-value pairs\n", __func__); - fclose(file); - gguf_free(ctx); - return NULL; - } - - ctx->infos = GGML_ALIGNED_MALLOC(ctx->header.n_tensors * sizeof(struct gguf_tensor_info)); + // read the tensor infos + { + ctx->infos = GGML_ALIGNED_MALLOC(ctx->header.n_tensors * sizeof(struct gguf_tensor_info)); - for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { - struct gguf_tensor_info * info = &ctx->infos[i]; + for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { + struct gguf_tensor_info * info = &ctx->infos[i]; - for (int j = 0; j < GGML_MAX_DIMS; ++j) { - info->ne[j] = 1; - } + for (int j = 0; j < GGML_MAX_DIMS; ++j) { + info->ne[j] = 1; + } - ok = ok && gguf_fread_str(&info->name, file, &offset); - ok = ok && gguf_fread_el (&info->n_dims, sizeof(info->n_dims), file, &offset); - for (uint32_t j = 0; j < info->n_dims; ++j) { - ok = ok && gguf_fread_el(&info->ne[j], sizeof(info->ne[j]), file, &offset); - } - //ok = ok && gguf_fread_el (&info->n_elms, sizeof(info->n_elms), file, &offset); - ok = ok && gguf_fread_el (&info->type, sizeof(info->type), file, &offset); - ok = ok && gguf_fread_el (&info->offset, sizeof(info->offset), file, &offset); + ok = ok && gguf_fread_str(&info->name, file, &offset); + ok = ok && gguf_fread_el (&info->n_dims, sizeof(info->n_dims), file, &offset); + for (uint32_t j = 0; j < info->n_dims; ++j) { + ok = ok && gguf_fread_el(&info->ne[j], sizeof(info->ne[j]), file, &offset); + } + //ok = ok && gguf_fread_el (&info->n_elms, sizeof(info->n_elms), file, &offset); + ok = ok && gguf_fread_el (&info->type, sizeof(info->type), file, &offset); + ok = ok && gguf_fread_el (&info->offset, sizeof(info->offset), file, &offset); - if (!ok) { - fprintf(stderr, "%s: failed to read tensor info\n", __func__); - fclose(file); - gguf_free(ctx); - return NULL; + if (!ok) { + fprintf(stderr, "%s: failed to read tensor info\n", __func__); + fclose(file); + gguf_free(ctx); + return NULL; + } } } @@ -18507,6 +18520,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p // TODO: determine new alignment from kv if available + // we require the data section to be aligned, so take into account any padding { const size_t offset_pad = offset % ctx->alignment; @@ -18516,38 +18530,46 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p } } + // store the current file offset - this is where the data section starts ctx->offset = offset; - ctx->size_data = 0; + // compute the total size of the data section, taking into account the alignment + { - for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { - struct gguf_tensor_info * info = &ctx->infos[i]; + ctx->size_data = 0; + for (uint32_t i = 0; i < ctx->header.n_tensors; ++i) { + struct gguf_tensor_info * info = &ctx->infos[i]; - const int64_t ne = - (int64_t) info->ne[0] * - (int64_t) info->ne[1] * - (int64_t) info->ne[2] * - (int64_t) info->ne[3]; + const int64_t ne = + (int64_t) info->ne[0] * + (int64_t) info->ne[1] * + (int64_t) info->ne[2] * + (int64_t) info->ne[3]; - if (ne % ggml_blck_size(info->type) != 0) { - fprintf(stderr, "%s: tensor '%s' number of elements (%" PRId64 ") is not a multiple of block size (%d)\n", - __func__, info->name.data, ne, ggml_blck_size(info->type)); - fclose(file); - gguf_free(ctx); - return NULL; - } + if (ne % ggml_blck_size(info->type) != 0) { + fprintf(stderr, "%s: tensor '%s' number of elements (%" PRId64 ") is not a multiple of block size (%d)\n", + __func__, info->name.data, ne, ggml_blck_size(info->type)); + fclose(file); + gguf_free(ctx); + return NULL; + } - const size_t size_cur = (ne*ggml_type_size(info->type))/ggml_blck_size(info->type); + const size_t size_cur = (ne*ggml_type_size(info->type))/ggml_blck_size(info->type); - ctx->size_data += GGML_PAD(size_cur, ctx->alignment); + ctx->size_data += GGML_PAD(size_cur, ctx->alignment); + } } - // load the tensor data - // TODO: simplify + // load the tensor data only if requested if (params.ctx != NULL) { + // if the provided gguf_context is no_alloc, then we create "empty" tensors and do not read the binary blob + // otherwise, we load the binary blob into the created ggml_context as well, and point the "data" members of + // the ggml_tensor structs to the appropriate locations in the binary blob + + // compute the exact size needed for the new ggml_context const size_t mem_size = params.no_alloc ? - (ctx->header.n_tensors + 1)*ggml_tensor_overhead() : + (ctx->header.n_tensors )*ggml_tensor_overhead() : (ctx->header.n_tensors + 1)*ggml_tensor_overhead() + ctx->size_data; struct ggml_init_params pdata = { @@ -18567,7 +18589,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p ok = ok && data != NULL; - // read the tensor data + // read the binary blob with the tensor data ok = ok && gguf_fread_el(data->data, ctx->size_data, file, &offset); if (!ok) { @@ -18602,6 +18624,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p break; } + // point the data member to the appropriate location in the binary blob using the tensor infos if (params.no_alloc == false) { //cur->data = (char *) data->data + ctx->infos[i].offset - ctx->offset; // offset from start of file cur->data = (char *) data->data + ctx->infos[i].offset; // offset from data @@ -18609,7 +18632,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p } if (!ok) { - fprintf(stderr, "%s: failed to create tensors\n", __func__); + fprintf(stderr, "%s: failed to read the tensor data\n", __func__); fclose(file); ggml_free(ctx_data); gguf_free(ctx); @@ -18619,13 +18642,6 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p ggml_set_no_alloc(ctx_data, params.no_alloc); } - if (!ok) { - fprintf(stderr, "%s: failed to read tensor data\n", __func__); - fclose(file); - gguf_free(ctx); - return NULL; - } - return ctx; } From c85d3178b3165137c1f7d0b454db2b2b5efd7b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Thu, 27 Jul 2023 10:29:29 +0300 Subject: [PATCH 014/242] refactor : reduce code duplication and better API (#2415) --- gguf.py | 57 +++++++++++++++++++++++++-------------------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/gguf.py b/gguf.py index dfd5ba5bf8490..991bbe2f32efc 100644 --- a/gguf.py +++ b/gguf.py @@ -71,63 +71,56 @@ def open(cls, path: str) -> "GGUFWriter": f = open(path, "wb") return cls(f) - def write_key(self, key: str, value_type: GGUFValueType): - encoded_key = key.encode("utf8") - self.buffered_writer.write(struct.pack(" Date: Thu, 27 Jul 2023 11:10:34 +0300 Subject: [PATCH 015/242] gguf : expose the gguf_type enum through the API for now --- examples/gguf/gguf.cpp | 13 ------------- ggml.c | 13 ------------- ggml.h | 14 ++++++++++++++ 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index a5c442ac581f5..d6a0691d016eb 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -7,19 +7,6 @@ #include #include -enum gguf_type { - GGUF_TYPE_UINT8 = 0, - GGUF_TYPE_INT8 = 1, - GGUF_TYPE_UINT16 = 2, - GGUF_TYPE_INT16 = 3, - GGUF_TYPE_UINT32 = 4, - GGUF_TYPE_INT32 = 5, - GGUF_TYPE_FLOAT32 = 6, - GGUF_TYPE_BOOL = 7, - GGUF_TYPE_STRING = 8, - GGUF_TYPE_ARRAY = 9, -}; - template static std::string to_string(const T & val) { std::stringstream ss; diff --git a/ggml.c b/ggml.c index b005fd8893e58..ebdb6536f81fa 100644 --- a/ggml.c +++ b/ggml.c @@ -18297,19 +18297,6 @@ size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, i //////////////////////////////////////////////////////////////////////////////// -enum gguf_type { - GGUF_TYPE_UINT8 = 0, - GGUF_TYPE_INT8 = 1, - GGUF_TYPE_UINT16 = 2, - GGUF_TYPE_INT16 = 3, - GGUF_TYPE_UINT32 = 4, - GGUF_TYPE_INT32 = 5, - GGUF_TYPE_FLOAT32 = 6, - GGUF_TYPE_BOOL = 7, - GGUF_TYPE_STRING = 8, - GGUF_TYPE_ARRAY = 9, -}; - struct gguf_str { uint32_t n; char * data; diff --git a/ggml.h b/ggml.h index e0abbbfdd81e3..91588895ce70e 100644 --- a/ggml.h +++ b/ggml.h @@ -1619,6 +1619,20 @@ extern "C" { // gguf // + // TODO: can be removed if the API is extended for writing + enum gguf_type { + GGUF_TYPE_UINT8 = 0, + GGUF_TYPE_INT8 = 1, + GGUF_TYPE_UINT16 = 2, + GGUF_TYPE_INT16 = 3, + GGUF_TYPE_UINT32 = 4, + GGUF_TYPE_INT32 = 5, + GGUF_TYPE_FLOAT32 = 6, + GGUF_TYPE_BOOL = 7, + GGUF_TYPE_STRING = 8, + GGUF_TYPE_ARRAY = 9, + }; + struct gguf_context; struct gguf_init_params { From d2b6ca13ad25a55e64bdd8287e773393fa54d212 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 27 Jul 2023 14:53:07 +0300 Subject: [PATCH 016/242] gguf : add array support --- examples/gguf/gguf.cpp | 87 +++++++++++++++++++++++++++++++++++------- ggml.c | 64 ++++++++++++++++++++++++++++--- ggml.h | 4 +- 3 files changed, 135 insertions(+), 20 deletions(-) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index d6a0691d016eb..c3494a343a6ec 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -29,7 +29,7 @@ void gguf_ex_write_u64(std::ofstream & fout, size_t val) { } template -void gguf_ex_write_param(std::ofstream & fout, const std::string & key, enum gguf_type type, const T & val) { +void gguf_ex_write_val(std::ofstream & fout, const std::string & key, enum gguf_type type, const T & val) { gguf_ex_write_str(fout, key); fout.write((const char *) &type, sizeof(type)); fout.write((const char *) &val, sizeof(val)); @@ -38,13 +38,65 @@ void gguf_ex_write_param(std::ofstream & fout, const std::string & key, enum ggu } template<> -void gguf_ex_write_param(std::ofstream & fout, const std::string & key, enum gguf_type type, const std::string & val) { +void gguf_ex_write_val(std::ofstream & fout, const std::string & key, enum gguf_type type, const std::string & val) { gguf_ex_write_str(fout, key); fout.write((const char *) &type, sizeof(type)); const int32_t n = val.size(); fout.write((const char *) &n, sizeof(n)); fout.write(val.c_str(), n); + + fprintf(stdout, "%s: write param: %s = %s\n", __func__, key.c_str(), val.c_str()); +} + +template +void gguf_ex_write_arr(std::ofstream & fout, const std::string & key, enum gguf_type type, const std::vector & val) { + gguf_ex_write_str(fout, key); + { + const enum gguf_type tarr = GGUF_TYPE_ARRAY; + fout.write((const char *) &tarr, sizeof(tarr)); + } + + const int32_t n = val.size(); + fout.write((const char *) &type, sizeof(type)); + fout.write((const char *) &n, sizeof(n)); + fout.write((const char *) val.data(), n * sizeof(T)); + + fprintf(stdout, "%s: write param: %s = [", __func__, key.c_str()); + for (int i = 0; i < n; ++i) { + fprintf(stdout, "%s", to_string(val[i]).c_str()); + if (i < n - 1) { + fprintf(stdout, ", "); + } + } + fprintf(stdout, "]\n"); +} + +template<> +void gguf_ex_write_arr(std::ofstream & fout, const std::string & key, enum gguf_type type, const std::vector & val) { + gguf_ex_write_str(fout, key); + { + const enum gguf_type tarr = GGUF_TYPE_ARRAY; + fout.write((const char *) &tarr, sizeof(tarr)); + } + + const int32_t n = val.size(); + fout.write((const char *) &type, sizeof(type)); + fout.write((const char *) &n, sizeof(n)); + for (int i = 0; i < n; ++i) { + const int32_t nstr = val[i].size(); + fout.write((const char *) &nstr, sizeof(nstr)); + fout.write(val[i].c_str(), nstr); + } + + fprintf(stdout, "%s: write param: %s = [", __func__, key.c_str()); + for (int i = 0; i < n; ++i) { + fprintf(stdout, "%s", val[i].c_str()); + if (i < n - 1) { + fprintf(stdout, ", "); + } + } + fprintf(stdout, "]\n"); } bool gguf_ex_write(const std::string & fname) { @@ -60,8 +112,9 @@ bool gguf_ex_write(const std::string & fname) { fout.write((const char *) &version, sizeof(version)); } + // NOTE: these have to match the output below! const int n_tensors = 10; - const int n_kv = 9; + const int n_kv = 12; fout.write((const char*) &n_tensors, sizeof(n_tensors)); fout.write((const char*) &n_kv, sizeof(n_kv)); @@ -70,17 +123,21 @@ bool gguf_ex_write(const std::string & fname) { // kv data { - gguf_ex_write_param< uint8_t>(fout, "some.parameter.uint8", GGUF_TYPE_UINT8, 0x12); - gguf_ex_write_param< int8_t>(fout, "some.parameter.int8", GGUF_TYPE_INT8, -0x13); - gguf_ex_write_param(fout, "some.parameter.uint16", GGUF_TYPE_UINT16, 0x1234); - gguf_ex_write_param< int16_t>(fout, "some.parameter.int16", GGUF_TYPE_INT16, -0x1235); - gguf_ex_write_param(fout, "some.parameter.uint32", GGUF_TYPE_UINT32, 0x12345678); - gguf_ex_write_param< int32_t>(fout, "some.parameter.int32", GGUF_TYPE_INT32, -0x12345679); + gguf_ex_write_val< uint8_t>(fout, "some.parameter.uint8", GGUF_TYPE_UINT8, 0x12); + gguf_ex_write_val< int8_t>(fout, "some.parameter.int8", GGUF_TYPE_INT8, -0x13); + gguf_ex_write_val(fout, "some.parameter.uint16", GGUF_TYPE_UINT16, 0x1234); + gguf_ex_write_val< int16_t>(fout, "some.parameter.int16", GGUF_TYPE_INT16, -0x1235); + gguf_ex_write_val(fout, "some.parameter.uint32", GGUF_TYPE_UINT32, 0x12345678); + gguf_ex_write_val< int32_t>(fout, "some.parameter.int32", GGUF_TYPE_INT32, -0x12345679); - gguf_ex_write_param (fout, "some.parameter.float32", GGUF_TYPE_FLOAT32, 0.123456789f); - gguf_ex_write_param (fout, "some.parameter.bool", GGUF_TYPE_BOOL, true); + gguf_ex_write_val (fout, "some.parameter.float32", GGUF_TYPE_FLOAT32, 0.123456789f); + gguf_ex_write_val (fout, "some.parameter.bool", GGUF_TYPE_BOOL, true); - gguf_ex_write_param(fout, "some.parameter.string", GGUF_TYPE_STRING, "hello world"); + gguf_ex_write_val(fout, "some.parameter.string", GGUF_TYPE_STRING, "hello world"); + + gguf_ex_write_arr (fout, "some.parameter.arr.i16", GGUF_TYPE_INT16, { 1, 2, 3, 4, }); + gguf_ex_write_arr (fout, "some.parameter.arr.f32", GGUF_TYPE_FLOAT32, { 3.145f, 2.718f, 1.414f, }); + gguf_ex_write_arr(fout, "some.parameter.arr.str", GGUF_TYPE_STRING, { "hello", "world", "!" }); } uint64_t offset_tensor = 0; @@ -203,13 +260,15 @@ bool gguf_ex_read_0(const std::string & fname) { fprintf(stdout, "%s: n_tensors: %d\n", __func__, n_tensors); for (int i = 0; i < n_tensors; ++i) { - const char * name = gguf_get_tensor_name(ctx, i); + const char * name = gguf_get_tensor_name (ctx, i); const size_t offset = gguf_get_tensor_offset(ctx, i); fprintf(stdout, "%s: tensor[%d]: name = %s, offset = %zu\n", __func__, i, name, offset); } } + gguf_free(ctx); + return true; } @@ -248,7 +307,7 @@ bool gguf_ex_read_1(const std::string & fname) { fprintf(stdout, "%s: n_tensors: %d\n", __func__, n_tensors); for (int i = 0; i < n_tensors; ++i) { - const char * name = gguf_get_tensor_name(ctx, i); + const char * name = gguf_get_tensor_name (ctx, i); const size_t offset = gguf_get_tensor_offset(ctx, i); fprintf(stdout, "%s: tensor[%d]: name = %s, offset = %zu\n", __func__, i, name, offset); diff --git a/ggml.c b/ggml.c index ebdb6536f81fa..96c7ebd347c1b 100644 --- a/ggml.c +++ b/ggml.c @@ -3698,7 +3698,6 @@ static const size_t GGML_TYPE_SIZE[GGML_TYPE_COUNT] = { }; static_assert(GGML_TYPE_COUNT == 19, "GGML_TYPE_SIZE is outdated"); - static const char * GGML_TYPE_NAME[GGML_TYPE_COUNT] = { [GGML_TYPE_F32] = "f32", [GGML_TYPE_F16] = "f16", @@ -18302,7 +18301,19 @@ struct gguf_str { char * data; }; -union gguf_value; +static const size_t GGUF_TYPE_SIZE[GGUF_TYPE_COUNT] = { + [GGUF_TYPE_UINT8] = sizeof(uint8_t), + [GGUF_TYPE_INT8] = sizeof(int8_t), + [GGUF_TYPE_UINT16] = sizeof(uint16_t), + [GGUF_TYPE_INT16] = sizeof(int16_t), + [GGUF_TYPE_UINT32] = sizeof(uint32_t), + [GGUF_TYPE_INT32] = sizeof(int32_t), + [GGUF_TYPE_FLOAT32] = sizeof(float), + [GGUF_TYPE_BOOL] = sizeof(bool), + [GGUF_TYPE_STRING] = sizeof(struct gguf_str), + [GGUF_TYPE_ARRAY] = 0, // undefined +}; +static_assert(GGUF_TYPE_COUNT == 10, "GGUF_TYPE_COUNT != 10"); union gguf_value { uint8_t uint8; @@ -18320,7 +18331,7 @@ union gguf_value { enum gguf_type type; uint32_t n; - union gguf_value * arr; + void * data; } arr; }; @@ -18457,8 +18468,35 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p case GGUF_TYPE_BOOL: ok = ok && gguf_fread_el (&kv->value.bool_, sizeof(kv->value.bool_), file, &offset); break; case GGUF_TYPE_STRING: ok = ok && gguf_fread_str(&kv->value.str, file, &offset); break; case GGUF_TYPE_ARRAY: - GGML_ASSERT("gguf: array type not implemented"); - break; + { + ok = ok && gguf_fread_el(&kv->value.arr.type, sizeof(kv->value.arr.type), file, &offset); + ok = ok && gguf_fread_el(&kv->value.arr.n, sizeof(kv->value.arr.n), file, &offset); + + switch (kv->value.arr.type) { + case GGUF_TYPE_UINT8: + case GGUF_TYPE_INT8: + case GGUF_TYPE_UINT16: + case GGUF_TYPE_INT16: + case GGUF_TYPE_UINT32: + case GGUF_TYPE_INT32: + case GGUF_TYPE_FLOAT32: + case GGUF_TYPE_BOOL: + { + kv->value.arr.data = malloc(kv->value.arr.n * GGUF_TYPE_SIZE[kv->value.arr.type]); + ok = ok && gguf_fread_el(kv->value.arr.data, kv->value.arr.n * GGUF_TYPE_SIZE[kv->value.arr.type], file, &offset); + } break; + case GGUF_TYPE_STRING: + { + kv->value.arr.data = malloc(kv->value.arr.n * sizeof(struct gguf_str)); + for (uint32_t j = 0; j < kv->value.arr.n; ++j) { + ok = ok && gguf_fread_str(&((struct gguf_str *) kv->value.arr.data)[j], file, &offset); + } + } break; + case GGUF_TYPE_ARRAY: + case GGUF_TYPE_COUNT: GGML_ASSERT(false && "invalid type"); + }; + } break; + case GGUF_TYPE_COUNT: GGML_ASSERT(false && "invalid type"); }; if (!ok) { @@ -18629,6 +18667,8 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p ggml_set_no_alloc(ctx_data, params.no_alloc); } + fclose(file); + return ctx; } @@ -18651,6 +18691,20 @@ void gguf_free(struct gguf_context * ctx) { free(kv->value.str.data); } } + + if (kv->type == GGUF_TYPE_ARRAY) { + if (kv->value.arr.data) { + if (kv->value.arr.type == GGUF_TYPE_STRING) { + for (uint32_t j = 0; j < kv->value.arr.n; ++j) { + struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[j]; + if (str->data) { + free(str->data); + } + } + } + free(kv->value.arr.data); + } + } } GGML_ALIGNED_FREE(ctx->header.kv); diff --git a/ggml.h b/ggml.h index 91588895ce70e..e857b3f14435f 100644 --- a/ggml.h +++ b/ggml.h @@ -1631,6 +1631,7 @@ extern "C" { GGUF_TYPE_BOOL = 7, GGUF_TYPE_STRING = 8, GGUF_TYPE_ARRAY = 9, + GGUF_TYPE_COUNT, // marks the end of the enum }; struct gguf_context; @@ -1664,7 +1665,8 @@ extern "C" { GGML_API float gguf_get_val_f32 (struct gguf_context * ctx, int i); GGML_API bool gguf_get_val_bool(struct gguf_context * ctx, int i); GGML_API const char * gguf_get_val_str (struct gguf_context * ctx, int i); - // TODO: arr + GGML_API int gguf_get_arr_n (struct gguf_context * ctx, int i); + GGML_API void gguf_get_arr_data(struct gguf_context * ctx, int i, void * data); GGML_API int gguf_get_n_tensors (struct gguf_context * ctx); GGML_API size_t gguf_get_tensor_offset(struct gguf_context * ctx, int i); From 158be8f7f4f0f34ef8057a87f0e35a3b2af2ed4d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 27 Jul 2023 15:37:06 +0300 Subject: [PATCH 017/242] gguf.py : some code style changes --- constants.py | 44 ++++++++-------- gguf.py | 142 +++++++++++++++++++++++++-------------------------- 2 files changed, 92 insertions(+), 94 deletions(-) diff --git a/constants.py b/constants.py index 7c7456403aaeb..3a97460e5dc5e 100644 --- a/constants.py +++ b/constants.py @@ -1,32 +1,32 @@ -GGUF_MAGIC = 0x47475546 +GGUF_MAGIC = 0x47475546 GGUF_VERSION = 1 # general -KEY_GENERAL_ARCHITECTURE = "general.architecture" +KEY_GENERAL_ARCHITECTURE = "general.architecture" KEY_GENERAL_QUANTIZATION_VERSION = "general.quantization_version" -KEY_GENERAL_NAME = "general.name" -KEY_GENERAL_AUTHOR = "general.author" -KEY_GENERAL_URL = "general.url" -KEY_GENERAL_DESCRIPTION = "general.description" -KEY_GENERAL_FILE_TYPE = "general.file_type" -KEY_GENERAL_LICENSE = "general.license" -KEY_GENERAL_SOURCE_URL = "general.source.url" -KEY_GENERAL_SOURCE_HF_REPO = "general.source.hugginface.repository" +KEY_GENERAL_NAME = "general.name" +KEY_GENERAL_AUTHOR = "general.author" +KEY_GENERAL_URL = "general.url" +KEY_GENERAL_DESCRIPTION = "general.description" +KEY_GENERAL_FILE_TYPE = "general.file_type" +KEY_GENERAL_LICENSE = "general.license" +KEY_GENERAL_SOURCE_URL = "general.source.url" +KEY_GENERAL_SOURCE_HF_REPO = "general.source.hugginface.repository" # LLM -KEY_LLM_CONTEXT_LENGTH = "{llm}.context_length" -KEY_LLM_EMBEDDING_LENGTH = "{llm}.embedding_length" -KEY_LLM_LAYER_COUNT = "{llm}.layer_count" -KEY_LLM_FEED_FORWARD_LENGTH = "{llm}.feed_forward_length" -KEY_LLM_USE_PARALLEL_RESIDUAL = "{llm}.use_parallel_residual" -KEY_LLM_TENSOR_DATA_LAYOUT = "{llm}.tensor_data_layout" +KEY_LLM_CONTEXT_LENGTH = "{llm}.context_length" +KEY_LLM_EMBEDDING_LENGTH = "{llm}.embedding_length" +KEY_LLM_LAYER_COUNT = "{llm}.layer_count" +KEY_LLM_FEED_FORWARD_LENGTH = "{llm}.feed_forward_length" +KEY_LLM_USE_PARALLEL_RESIDUAL = "{llm}.use_parallel_residual" +KEY_LLM_TENSOR_DATA_LAYOUT = "{llm}.tensor_data_layout" # attention -KEY_ATTENTION_HEAD_COUNT = "{llm}.attention.head_count" -KEY_ATTENTION_HEAD_COUNT_KV = "{llm}.attention.head_count_kv" -KEY_ATTENTION_MAX_ALIBI_BIAS = "{llm}.attention.max_alibi_bias" -KEY_ATTENTION_CLAMP_KQV = "{llm}.attention.clamp_kqv" +KEY_ATTENTION_HEAD_COUNT = "{llm}.attention.head_count" +KEY_ATTENTION_HEAD_COUNT_KV = "{llm}.attention.head_count_kv" +KEY_ATTENTION_MAX_ALIBI_BIAS = "{llm}.attention.max_alibi_bias" +KEY_ATTENTION_CLAMP_KQV = "{llm}.attention.clamp_kqv" # RoPE -KEY_ROPE_DIMENSION_COUNT = "{llm}.rope.dimension_count" -KEY_ROPE_SCALE = "{llm}.rope.scale" +KEY_ROPE_DIMENSION_COUNT = "{llm}.rope.dimension_count" +KEY_ROPE_SCALE = "{llm}.rope.scale" diff --git a/gguf.py b/gguf.py index 991bbe2f32efc..764ae9a9df27c 100644 --- a/gguf.py +++ b/gguf.py @@ -6,14 +6,13 @@ """ import struct +import constants from enum import IntEnum from typing import List, Any -import constants - class GGMLQuantizationType(IntEnum): - F32 = 0 - F16 = 1 + F32 = 0 + F16 = 1 QR_0 = 2 Q4_1 = 3 # Q4_2 = 4 # support has been removed @@ -31,31 +30,30 @@ class GGMLQuantizationType(IntEnum): class GGUFValueType(IntEnum): - UINT8 = 0 - INT8 = 1 - UINT16 = 2 - INT16 = 3 - UINT32 = 4 - INT32 = 5 + UINT8 = 0 + INT8 = 1 + UINT16 = 2 + INT16 = 3 + UINT32 = 4 + INT32 = 5 FLOAT32 = 6 - BOOL = 7 - STRING = 8 - ARRAY = 9 + BOOL = 7 + STRING = 8 + ARRAY = 9 @staticmethod - def get_type(value): - if isinstance(value, str): + def get_type(val): + if isinstance(val, str): return GGUFValueType.STRING - elif isinstance(value, list): + elif isinstance(val, list): return GGUFValueType.ARRAY - elif isinstance(value, float): + elif isinstance(val, float): return GGUFValueType.FLOAT32 - elif isinstance(value, bool): + elif isinstance(val, bool): return GGUFValueType.BOOL else: return GGUFValueType.INT32 - class GGUFWriter: def __init__(self, buffered_writer): self.buffered_writer = buffered_writer @@ -72,81 +70,81 @@ def open(cls, path: str) -> "GGUFWriter": return cls(f) def write_key(self, key: str): - self.write_value(key, GGUFValueType.STRING) + self.write_val(key, GGUFValueType.STRING) - def write_uint8(self, key: str, value: int): + def write_uint8(self, key: str, val: int): self.write_key(key) - self.write_value(value, GGUFValueType.UINT8) + self.write_val(val, GGUFValueType.UINT8) - def write_int8(self, key: str, value: int): + def write_int8(self, key: str, val: int): self.write_key(key) - self.write_value(value, GGUFValueType.INT8) + self.write_val(val, GGUFValueType.INT8) - def write_uint16(self, key: str, value: int): + def write_uint16(self, key: str, val: int): self.write_key(key) - self.write_value(value, GGUFValueType.UINT16) + self.write_val(val, GGUFValueType.UINT16) - def write_int16(self, key: str, value: int): + def write_int16(self, key: str, val: int): self.write_key(key) - self.write_value(value, GGUFValueType.INT16) + self.write_val(val, GGUFValueType.INT16) - def write_uint32(self, key: str, value: int): + def write_uint32(self, key: str, val: int): self.write_key(key) - self.write(value, GGUFValueType.UINT32) + self.write_val(val, GGUFValueType.UINT32) - def write_int32(self, key: str, value: int): + def write_int32(self, key: str, val: int): self.write_key(key) - self.write_value(value, GGUFValueType.INT32) + self.write_val(val, GGUFValueType.INT32) - def write_float32(self, key: str, value: float): + def write_float32(self, key: str, val: float): self.write_key(key) - self.write_value(value, GGUFValueType.FLOAT32) + self.write_val(val, GGUFValueType.FLOAT32) - def write_bool(self, key: str, value: bool): + def write_bool(self, key: str, val: bool): self.write_key(key) - self.write_value(value, GGUFValueType.BOOL) + self.write_val(val, GGUFValueType.BOOL) - def write_string(self, key: str, value: str): + def write_string(self, key: str, val: str): self.write_key(key) - self.write_value(value, GGUFValueType.STRING) + self.write_val(val, GGUFValueType.STRING) - def write_array(self, key: str, value: list): - if not isinstance(value, list): + def write_array(self, key: str, val: list): + if not isinstance(val, list): raise ValueError("Value must be a list for array type") self.write_key(key) - self.write_value(value, GGUFValueType.ARRAY) - - def write_value(self: str, value: Any, value_type: GGUFValueType = None): - if value_type is None: - value_type = GGUFValueType.get_type(value) - - self.buffered_writer.write(struct.pack(" Date: Thu, 27 Jul 2023 15:56:53 +0300 Subject: [PATCH 018/242] convert.py : start a new simplified implementation by removing old stuff --- convert-new.py | 961 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 961 insertions(+) create mode 100755 convert-new.py diff --git a/convert-new.py b/convert-new.py new file mode 100755 index 0000000000000..dacad693fea1c --- /dev/null +++ b/convert-new.py @@ -0,0 +1,961 @@ +#!/usr/bin/env python + +import argparse +import concurrent.futures +import copy +import enum +import faulthandler +import functools +import io +import itertools +import json +import math +import mmap +import pickle +import re +import signal +import struct +import sys +import zipfile +import numpy as np + +from abc import ABCMeta, abstractmethod +from dataclasses import dataclass +from pathlib import Path +from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Literal, Optional, Sequence, Tuple, TypeVar, Union) +from sentencepiece import SentencePieceProcessor # type: ignore + +if TYPE_CHECKING: + from typing_extensions import TypeAlias + +if hasattr(faulthandler, 'register') and hasattr(signal, 'SIGUSR1'): + faulthandler.register(signal.SIGUSR1) + +NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]' + +@dataclass(frozen=True) +class UnquantizedDataType: + name: str + +DT_F16 = UnquantizedDataType('F16') +DT_F32 = UnquantizedDataType('F32') +DT_I32 = UnquantizedDataType('I32') +DT_BF16 = UnquantizedDataType('BF16') + +DataType = Union[UnquantizedDataType] + +DATA_TYPE_TO_FTYPE: Dict[DataType, int] = { + DT_F32: 0, + DT_F16: 1, +} + +FTYPE_TO_DATA_TYPE: Dict[int, DataType] = \ + {ftype: dtype for (dtype, ftype) in DATA_TYPE_TO_FTYPE.items()} + +DATA_TYPE_TO_NUMPY: Dict[DataType, 'np.dtype[Any]'] = { + DT_BF16: np.dtype(np.uint16), + DT_F16: np.dtype(np.float16), + DT_F32: np.dtype(np.float32), + DT_I32: np.dtype(np.int32), +} + +NUMPY_TYPE_TO_DATA_TYPE: Dict['np.dtype[Any]', DataType] = \ + {dtype: data_type for (data_type, dtype) in DATA_TYPE_TO_NUMPY.items()} + +class GGMLFileType(enum.Enum): + AllF32 = 0 + MostlyF16 = 1 # except 1d tensors + + def type_for_tensor(self, name: str, tensor: 'LazyTensor') -> DataType: + if len(tensor.shape) == 1: + # 1D tensors are always F32. + return DT_F32 + elif self == GGMLFileType.AllF32: + return DT_F32 + elif self == GGMLFileType.MostlyF16: + return DT_F16 + else: + raise ValueError(self) + +# TODO: this is LLaMA specific +def make_tensors_list() -> List[str]: + ret = [ + 'tok_embeddings.weight', + 'norm.weight', + 'output.weight', + ] + for i in range(80): # maximum number of layer + ret += [ + f'layers.{i}.attention.wq.weight', + f'layers.{i}.attention.wk.weight', + f'layers.{i}.attention.wv.weight', + f'layers.{i}.attention.wo.weight', + f'layers.{i}.attention_norm.weight', + f'layers.{i}.feed_forward.w1.weight', + f'layers.{i}.feed_forward.w2.weight', + f'layers.{i}.feed_forward.w3.weight', + f'layers.{i}.ffn_norm.weight', + ] + return ret + +# TODO: this should be generalized for non-LLaMA models +TENSORS_LIST = make_tensors_list() +TENSORS_SET = set(TENSORS_LIST) + +def find_n_mult(n_ff: int, n_embd: int) -> int: + # hardcoded magic range + for n_mult in range(256, 1, -1): + calc_ff = (((8*n_embd) // 3 + n_mult - 1) // n_mult)*n_mult + if calc_ff == n_ff: + return n_mult + raise Exception(f"failed to find n_mult for (n_ff={n_ff}, n_embd={n_embd}).") + + +@dataclass +class Params: + n_vocab: int + n_embd: int + n_mult: int + n_head: int + n_layer: int + + @staticmethod + def guessed(model: 'LazyModel') -> 'Params': + # try transformer naming first + n_vocab, n_embd = model["model.embed_tokens.weight"].shape if "model.embed_tokens.weight" in model else model["tok_embeddings.weight"].shape + + # try transformer naming first + if "model.layers.0.self_attn.q_proj.weight" in model: + n_layer=next(i for i in itertools.count() if f"model.layers.{i}.self_attn.q_proj.weight" not in model) + elif "model.layers.0.self_attn.W_pack.weight" in model: # next: try baichuan naming + n_layer=next(i for i in itertools.count() if f"model.layers.{i}.self_attn.W_pack.weight" not in model) + else: + n_layer=next(i for i in itertools.count() if f"layers.{i}.attention.wq.weight" not in model) + + if n_layer < 1: + raise Exception("failed to guess 'n_layer'. This model is unknown or unsupported.\n" + "Suggestion: provide 'config.json' of the model in the same directory containing model files.") + + n_head=n_embd // 128 # guessed + + return Params( + n_vocab = n_vocab, + n_embd = n_embd, + n_mult = 256, + n_head = n_head, + n_layer = n_layer, + ) + + @staticmethod + def loadHFTransformerJson(model: 'LazyModel', config_path: 'Path') -> 'Params': + config = json.load(open(config_path)) + + n_vocab = config["vocab_size"]; + n_embd = config["hidden_size"]; + n_head = config["num_attention_heads"]; + n_layer = config["num_hidden_layers"]; + n_ff = config["intermediate_size"]; + + n_mult = find_n_mult(n_ff, n_embd); + + return Params( + n_vocab = n_vocab, + n_embd = n_embd, + n_mult = n_mult, + n_head = n_head, + n_layer = n_layer, + ) + + # LLaMA v2 70B params.json + # {"dim": 8192, "multiple_of": 4096, "ffn_dim_multiplier": 1.3, "n_heads": 64, "n_kv_heads": 8, "n_layers": 80, "norm_eps": 1e-05, "vocab_size": -1 + @staticmethod + def loadOriginalParamsJson(model: 'LazyModel', config_path: 'Path') -> 'Params': + config = json.load(open(config_path)) + + n_vocab = config["vocab_size"]; + n_embd = config["dim"]; + n_head = config["n_heads"]; + n_layer = config["n_layers"]; + n_mult = config["multiple_of"]; + + if n_vocab == -1: + n_vocab = model["tok_embeddings.weight"].shape[0] + + return Params( + n_vocab = n_vocab, + n_embd = n_embd, + n_mult = n_mult, + n_head = n_head, + n_layer = n_layer, + ) + + @staticmethod + def load(model_plus: 'ModelPlus') -> 'Params': + hf_config_path = model_plus.paths[0].parent / "config.json" + orig_config_path = model_plus.paths[0].parent / "params.json" + + if hf_config_path.exists(): + params = Params.loadHFTransformerJson(model_plus.model, hf_config_path) + elif orig_config_path.exists(): + params = Params.loadOriginalParamsJson(model_plus.model, orig_config_path) + else: + params = Params.guessed(model_plus.model) + + print(f'params: n_vocab:{params.n_vocab} n_embd:{params.n_embd} n_mult:{params.n_mult} n_head:{params.n_head} n_layer:{params.n_layer}') + return params + + +class SentencePieceVocab: + def __init__(self, fname_tokenizer: Path, fname_added_tokens: Optional[Path], vocabtype: Optional[str]) -> None: + self.vocabtype = vocabtype + if self.vocabtype == "bpe": + self.sentencepiece_tokenizer = json.loads(open(str(fname_tokenizer)).read()) + else: + self.sentencepiece_tokenizer = SentencePieceProcessor(str(fname_tokenizer)) + + added_tokens: Dict[str, int] + if fname_added_tokens is not None: + added_tokens = json.load(open(fname_added_tokens)) + else: + added_tokens = {} + + if self.vocabtype == "bpe": + vocab_size: int = len(self.sentencepiece_tokenizer) + else: + vocab_size: int = self.sentencepiece_tokenizer.vocab_size() + + expected_ids = list(range(vocab_size, vocab_size + len(added_tokens))) + actual_ids = sorted(added_tokens.values()) + if expected_ids != actual_ids: + raise Exception(f"Expected added token IDs to be sequential and start at {len(added_tokens)}; got {actual_ids}") + + items = sorted(added_tokens.items(), key=lambda text_idx: text_idx[1]) + self.added_tokens_list = [text for (text, idx) in items] + self.vocab_size_base: int = vocab_size + self.vocab_size: int = self.vocab_size_base + len(self.added_tokens_list) + self.fname_tokenizer = fname_tokenizer + self.fname_added_tokens = fname_added_tokens + + def sentencepiece_tokens(self) -> Iterable[Tuple[bytes, float]]: + tokenizer = self.sentencepiece_tokenizer + if self.vocabtype == "bpe": + from transformers.models.gpt2 import tokenization_gpt2 + byte_encoder = tokenization_gpt2.bytes_to_unicode() + byte_decoder = {v: k for k, v in byte_encoder.items()} + for i, item in enumerate(tokenizer): + text: bytes + text = b''.join([x.to_bytes(1, byteorder='big') for x in [byte_decoder[y] for y in item]]) + score: float = -i + yield text, score + else: + for i in range(tokenizer.vocab_size()): + text: bytes + if tokenizer.is_unknown(i): + text = " \u2047 ".encode("utf-8") + elif tokenizer.is_control(i): + text = b"" + elif tokenizer.is_byte(i): + piece = tokenizer.id_to_piece(i) + if len(piece) != 6: + raise Exception(f"Invalid token: {piece}") + byte_value = int(piece[3:-1], 16) + text = struct.pack("B", byte_value) + else: + text = tokenizer.id_to_piece(i).replace("\u2581", " ").encode("utf-8") + score: float = tokenizer.get_score(i) + yield text, score + + def added_tokens(self) -> Iterable[Tuple[bytes, float]]: + for text in self.added_tokens_list: + score = -1000.0 + yield text.encode("utf-8"), score + + def all_tokens(self) -> Iterable[Tuple[bytes, float]]: + yield from self.sentencepiece_tokens() + yield from self.added_tokens() + + def __repr__(self) -> str: + return f"" + + +class GGMLVocab: + def __init__(self, tokens: List[Tuple[bytes, float]]): + self.tokens = tokens + self.vocab_size = len(tokens) + + def all_tokens(self) -> Iterable[Tuple[bytes, float]]: + return self.tokens + + def __repr__(self) -> str: + return f"" + + +Vocab = Union[SentencePieceVocab, GGMLVocab] + + +def permute(weights: NDArray, n_head: int) -> NDArray: + return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) + .swapaxes(1, 2) + .reshape(weights.shape)) + + +class Tensor(metaclass=ABCMeta): + data_type: DataType + + @abstractmethod + def astype(self, data_type: DataType) -> 'Tensor': ... + @abstractmethod + def permute(self, n_head: int) -> 'Tensor': ... + @abstractmethod + def permute_part(self, n_part: int, n_head: int) -> 'UnquantizedTensor': ... + @abstractmethod + def part(self, n_part: int) -> 'UnquantizedTensor': ... + @abstractmethod + def to_ggml(self) -> 'GGMLCompatibleTensor': ... + + +def bf16_to_fp32(bf16_arr: np.ndarray) -> np.ndarray: + assert bf16_arr.dtype == np.uint16, f"Input array should be of dtype uint16, but got {bf16_arr.dtype}" + fp32_arr = bf16_arr.astype(np.uint32) << 16 + return fp32_arr.view(np.float32) + + +class UnquantizedTensor(Tensor): + def __init__(self, ndarray: NDArray) -> None: + assert isinstance(ndarray, np.ndarray) + self.ndarray = ndarray + self.data_type = NUMPY_TYPE_TO_DATA_TYPE[ndarray.dtype] + + def astype(self, data_type: DataType) -> Tensor: + dtype = DATA_TYPE_TO_NUMPY[data_type] + if self.data_type == DT_BF16: + self.ndarray = bf16_to_fp32(self.ndarray) + return UnquantizedTensor(self.ndarray.astype(dtype)) + + def to_ggml(self) -> 'UnquantizedTensor': + return self + + def permute_part(self, n_part: int, n_head: int) -> 'UnquantizedTensor': + r = self.ndarray.shape[0] // 3 + return UnquantizedTensor(permute(self.ndarray[r * n_part : r * n_part + r, ...], n_head)) + + def part(self, n_part: int) -> 'UnquantizedTensor': + r = self.ndarray.shape[0] // 3 + return UnquantizedTensor(self.ndarray[r * n_part : r * n_part + r, ...]) + + def permute(self, n_head: int) -> 'UnquantizedTensor': + return UnquantizedTensor(permute(self.ndarray, n_head)) + + +def load_unquantized(lazy_tensor: 'LazyTensor', expected_dtype: Any = None, convert: bool = False) -> NDArray: + tensor = lazy_tensor.load() + assert isinstance(tensor, UnquantizedTensor) + + # double-check: + actual_shape = list(tensor.ndarray.shape) + assert actual_shape == lazy_tensor.shape, (actual_shape, lazy_tensor.shape) + if expected_dtype is not None and expected_dtype != tensor.ndarray.dtype: + if convert: + tensor.ndarray = tensor.ndarray.astype(expected_dtype) + else: + raise ValueError(f'expected this tensor to have dtype {expected_dtype}, got {tensor.ndarray.dtype}') + + return tensor.ndarray + + +GGMLCompatibleTensor = Union[UnquantizedTensor] + + +class DeferredPermutedTensor(Tensor): + def __init__(self, base: Tensor, n_head: int) -> None: + self.base = base + self.n_head = n_head + self.data_type = self.base.data_type + + def astype(self, data_type: DataType) -> Tensor: + return self.base.astype(data_type).permute(self.n_head) + + def to_ggml(self) -> GGMLCompatibleTensor: + return self.base.to_ggml().permute(self.n_head) + + def permute(self, n_head: int) -> Tensor: + raise Exception("shouldn't permute twice") + + +@dataclass +class LazyTensor: + _load: Callable[[], Tensor] + shape: List[int] + data_type: DataType + description: str + + def load(self) -> Tensor: + ret = self._load() + assert ret.data_type == self.data_type, (self.data_type, ret.data_type, self.description) + return ret + + def astype(self, data_type: DataType) -> 'LazyTensor': + self.validate_conversion_to(data_type) + + def load() -> Tensor: + return self.load().astype(data_type) + return LazyTensor(load, self.shape, data_type, f'convert({data_type}) {self.description}') + + def validate_conversion_to(self, data_type: DataType) -> None: + if data_type == self.data_type: + return + + +LazyModel = Dict[str, LazyTensor] + + +@dataclass +class ModelPlus: + model: LazyModel + paths: List[Path] # Where this was read from. + format: Literal['ggml', 'torch', 'safetensors'] + vocab: Optional[Vocab] # For GGML models (which have vocab built in), the vocab. + + +def merge_sharded(models: List[LazyModel]) -> LazyModel: + # Original LLaMA models have each file contain one part of each tensor. + # Use a dict instead of a set to preserve order. + names = {name: None for model in models for name in model} + + def convert(name: str) -> LazyTensor: + lazy_tensors: List[LazyTensor] = [model[name] for model in models] + if len(lazy_tensors) == 1: + # only one file; don't go through this procedure since there might + # be quantized tensors + return lazy_tensors[0] + if len(lazy_tensors[0].shape) == 1: + # the tensor is just duplicated in every file + return lazy_tensors[0] + if name.startswith('tok_embeddings.') or \ + name.endswith('.attention.wo.weight') or \ + name.endswith('.feed_forward.w2.weight'): + # split by columns + axis = 1 + else: + # split by rows + axis = 0 + concatenated_shape = list(lazy_tensors[0].shape) + concatenated_shape[axis] = sum(tensor.shape[axis] for tensor in lazy_tensors) + + def load() -> UnquantizedTensor: + ndarrays = [load_unquantized(tensor) for tensor in lazy_tensors] + concatenated: NDArray = np.concatenate(ndarrays, axis=axis) + return UnquantizedTensor(concatenated) + description = 'concatenated[[' + '] | ['.join(lt.description for lt in lazy_tensors) + ']]' + return LazyTensor(load, concatenated_shape, lazy_tensors[0].data_type, description) + return {name: convert(name) for name in names} + + +def merge_multifile_models(models_plus: List[ModelPlus]) -> ModelPlus: + formats = set(mp.format for mp in models_plus) + assert len(formats) == 1, "different formats?" + format = formats.pop() + paths = [path for mp in models_plus for path in mp.paths] + # Use the first non-None vocab, if any. + try: + vocab = next(mp.vocab for mp in models_plus if mp.vocab is not None) + except StopIteration: + vocab = None + + if any("model.embed_tokens.weight" in mp.model for mp in models_plus): + # Transformers models put different tensors in different files, but + # don't split indivdual tensors between files. + model: LazyModel = {} + for mp in models_plus: + model.update(mp.model) + else: + model = merge_sharded([mp.model for mp in models_plus]) + + return ModelPlus(model, paths, format, vocab) + + +def permute_lazy(lazy_tensor: LazyTensor, n_head: int) -> LazyTensor: + def load() -> Tensor: + return lazy_tensor.load().permute(n_head) + return LazyTensor(load, lazy_tensor.shape, lazy_tensor.data_type, f'permute({n_head}) ' + lazy_tensor.description) + +def permute_part_lazy(lazy_tensor: LazyTensor, n_part: int, n_head: int) -> LazyTensor: + def load() -> Tensor: + return lazy_tensor.load().permute_part(n_part, n_head) + s = lazy_tensor.shape.copy() + s[0] = s[0] // 3 + return LazyTensor(load, s, lazy_tensor.data_type, f'permute({n_head}) ' + lazy_tensor.description) + +def part_lazy(lazy_tensor: LazyTensor, n_part: int) -> LazyTensor: + def load() -> Tensor: + return lazy_tensor.load().part(n_part) + s = lazy_tensor.shape.copy() + s[0] = s[0] // 3 + return LazyTensor(load, s, lazy_tensor.data_type, 'part ' + lazy_tensor.description) + +def convert_transformers_to_orig(model: LazyModel, params: Params) -> LazyModel: + out: LazyModel = {} + out["tok_embeddings.weight"] = model["model.embed_tokens.weight"] + out["norm.weight"] = model["model.norm.weight"] + out["output.weight"] = model["lm_head.weight"] + + for i in itertools.count(): + if f"model.layers.{i}.self_attn.q_proj.weight" in model: + out[f"layers.{i}.attention.wq.weight"] = permute_lazy(model[f"model.layers.{i}.self_attn.q_proj.weight"], params.n_head) + out[f"layers.{i}.attention.wk.weight"] = permute_lazy(model[f"model.layers.{i}.self_attn.k_proj.weight"], params.n_head) + out[f"layers.{i}.attention.wv.weight"] = model[f"model.layers.{i}.self_attn.v_proj.weight"] + elif f"model.layers.{i}.self_attn.W_pack.weight" in model: + out[f"layers.{i}.attention.wq.weight"] = permute_part_lazy(model[f"model.layers.{i}.self_attn.W_pack.weight"], 0, params.n_head) + out[f"layers.{i}.attention.wk.weight"] = permute_part_lazy(model[f"model.layers.{i}.self_attn.W_pack.weight"], 1, params.n_head) + out[f"layers.{i}.attention.wv.weight"] = part_lazy(model[f"model.layers.{i}.self_attn.W_pack.weight"], 2) + else: + break + + out[f"layers.{i}.attention.wo.weight"] = model[f"model.layers.{i}.self_attn.o_proj.weight"] + + out[f"layers.{i}.feed_forward.w1.weight"] = model[f"model.layers.{i}.mlp.gate_proj.weight"] + out[f"layers.{i}.feed_forward.w2.weight"] = model[f"model.layers.{i}.mlp.down_proj.weight"] + out[f"layers.{i}.feed_forward.w3.weight"] = model[f"model.layers.{i}.mlp.up_proj.weight"] + + out[f"layers.{i}.attention_norm.weight"] = model[f"model.layers.{i}.input_layernorm.weight"] + out[f"layers.{i}.ffn_norm.weight"] = model[f"model.layers.{i}.post_attention_layernorm.weight"] + return out + + +# Functionality that simulates `torch.load` but where individual tensors are +# only loaded into memory on demand, not all at once. +# PyTorch can't do this natively as of time of writing: +# - https://github.com/pytorch/pytorch/issues/64327 +# This allows us to de-shard without multiplying RAM usage, and also +# conveniently drops the PyTorch dependency (though we still need numpy). + + +@dataclass +class LazyStorageKind: + data_type: DataType + + +@dataclass +class LazyStorage: + load: Callable[[int, int], NDArray] + kind: LazyStorageKind + description: str + + +class LazyUnpickler(pickle.Unpickler): + def __init__(self, fp: IO[bytes], data_base_path: str, zip_file: zipfile.ZipFile): + super().__init__(fp) + self.data_base_path = data_base_path + self.zip_file = zip_file + + def persistent_load(self, pid: Any) -> Any: + assert pid[0] == 'storage' + assert isinstance(pid[1], LazyStorageKind) + data_type = pid[1].data_type + filename_stem = pid[2] + filename = self.data_base_path + '/' + filename_stem + info = self.zip_file.getinfo(filename) + + def load(offset: int, elm_count: int) -> NDArray: + dtype = DATA_TYPE_TO_NUMPY.get(data_type) + if dtype is None: + raise Exception("tensor stored in unsupported format") + fp = self.zip_file.open(info) + fp.seek(offset * dtype.itemsize) + size = elm_count * dtype.itemsize + data = fp.read(size) + assert len(data) == size + return np.frombuffer(data, dtype) + description = f'storage data_type={data_type} path-in-zip={filename} path={self.zip_file.filename}' + return LazyStorage(load=load, kind=pid[1], description=description) + + # @staticmethod + def lazy_rebuild_tensor_v2(storage: Any, storage_offset: Any, size: Any, stride: Any, + # pyright: ignore[reportSelfClsParameterName] + requires_grad: Any, backward_hooks: Any, metadata: Any = None) -> LazyTensor: + assert isinstance(storage, LazyStorage) + + def load() -> UnquantizedTensor: + elm_count = stride[0] * size[0] + return UnquantizedTensor(storage.load(storage_offset, elm_count).reshape(size)) + description = f'pickled storage_offset={storage_offset} in {storage.description}' + return LazyTensor(load, list(size), storage.kind.data_type, description) + + # @staticmethod + def rebuild_from_type_v2(func, new_type, args, state): + return func(*args) + + CLASSES: Dict[Any, Any] = { + ('torch._tensor', '_rebuild_from_type_v2'): rebuild_from_type_v2, + ('torch._utils', '_rebuild_tensor_v2'): lazy_rebuild_tensor_v2, + ('torch', 'BFloat16Storage'): LazyStorageKind(DT_BF16), + ('torch', 'HalfStorage'): LazyStorageKind(DT_F16), + ('torch', 'FloatStorage'): LazyStorageKind(DT_F32), + ('torch', 'IntStorage'): LazyStorageKind(DT_I32), + ('torch', 'Tensor'): LazyTensor, + } + + def find_class(self, module: str, name: str) -> Any: + if not module.startswith('torch'): + return super().find_class(module, name) + return self.CLASSES[(module, name)] + + +def lazy_load_torch_file(outer_fp: IO[bytes], path: Path) -> ModelPlus: + zf = zipfile.ZipFile(outer_fp) + pickle_paths = [name for name in zf.namelist() if name.endswith('.pkl')] + assert len(pickle_paths) == 1, pickle_paths + pickle_fp = zf.open(pickle_paths[0], 'r') + unpickler = LazyUnpickler(pickle_fp, + data_base_path=pickle_paths[0][:-4], + zip_file=zf) + model = unpickler.load() + as_dict = dict(model.items()) + return ModelPlus(model=as_dict, paths=[path], format='torch', vocab=None) + + +SAFETENSORS_DATA_TYPES: Dict[str, DataType] = { + 'BF16': DT_BF16, + 'F16': DT_F16, + 'F32': DT_F32, + 'I32': DT_I32, +} + + +def lazy_load_safetensors_file(fp: IO[bytes], path: Path) -> ModelPlus: + header_size, = struct.unpack(' LazyTensor: + data_type = SAFETENSORS_DATA_TYPES[info['dtype']] + numpy_dtype = DATA_TYPE_TO_NUMPY[data_type] + shape: List[int] = info['shape'] + begin, end = info['data_offsets'] + assert 0 <= begin <= end <= len(byte_buf) + assert end - begin == math.prod(shape) * numpy_dtype.itemsize + buf = byte_buf[begin:end] + + def load() -> UnquantizedTensor: + return UnquantizedTensor(np.frombuffer(buf, dtype=numpy_dtype).reshape(shape)) + description = f'safetensors begin={begin} end={end} type={data_type} path={path}' + return LazyTensor(load, shape, data_type, description) + model = {name: convert(info) for (name, info) in header.items() if name != '__metadata__'} + return ModelPlus(model=model, paths=[path], format='safetensors', vocab=None) + + +def must_read(fp: IO[bytes], length: int) -> bytes: + ret = fp.read(length) + if len(ret) < length: + raise Exception("unexpectedly reached end of file") + return ret + + +@functools.lru_cache(maxsize=None) +def lazy_load_file(path: Path) -> ModelPlus: + fp = open(path, 'rb') + first8 = fp.read(8) + fp.seek(0) + if first8[:2] == b'PK': + # A zip file, i.e. PyTorch format + return lazy_load_torch_file(fp, path) + elif struct.unpack(' Iterable[Out]: + '''Parallel map, but with backpressure. If the caller doesn't call `next` + fast enough, this will stop calling `func` at some point rather than + letting results pile up in memory. Specifically, there is a max of one + output value buffered per thread.''' + with concurrent.futures.ThreadPoolExecutor() as executor: + futures: List[concurrent.futures.Future[Out]] = [] + items_rev = list(iterable)[::-1] + for i in range(min(concurrency, len(items_rev))): + futures.append(executor.submit(func, items_rev.pop())) + while futures: + result = futures.pop(0).result() + if items_rev: + futures.append(executor.submit(func, items_rev.pop())) + yield result + + +def check_vocab_size(params: Params, vocab: Vocab) -> None: + if params.n_vocab != vocab.vocab_size: + # GGMLVocab comes from the same file as the model so shouldn't mismatch: + assert isinstance(vocab, SentencePieceVocab) + if params.n_vocab == vocab.vocab_size_base: + print("Ignoring added_tokens.json since model matches vocab size without it.") + vocab.added_tokens_list = [] + vocab.vocab_size = vocab.vocab_size_base + return + msg = f"Vocab size mismatch (model has {params.n_vocab}, but {vocab.fname_tokenizer}" + if vocab.fname_added_tokens is not None: + msg += f" combined with {vocab.fname_added_tokens}" + msg += f" has {vocab.vocab_size})." + if vocab.vocab_size < params.n_vocab < vocab.vocab_size + 20 and vocab.fname_added_tokens is None: + msg += f" Most likely you are missing added_tokens.json (should be in {vocab.fname_tokenizer.parent})." + raise Exception(msg) + + +class OutputFile: + def __init__(self, fname_out: Path) -> None: + self.fout = open(fname_out, "wb") + + def write_file_header(self, params: Params, file_type: GGMLFileType) -> None: + self.fout.write(b"ggjt"[::-1]) # magic + values = [ + 1, # file version + params.n_vocab, + params.n_embd, + params.n_mult, + params.n_head, + params.n_layer, + params.n_embd // params.n_head, # rot (obsolete) + file_type.value, + ] + self.fout.write(struct.pack("i" * len(values), *values)) + + def write_tensor_header(self, name: str, shape: Sequence[int], data_type: DataType) -> None: + sname = name.encode('utf-8') + self.fout.write(struct.pack("iii", len(shape), len(sname), DATA_TYPE_TO_FTYPE[data_type])) + self.fout.write(struct.pack("i" * len(shape), *shape[::-1])) + self.fout.write(sname) + self.fout.seek((self.fout.tell() + 31) & -32) + + def write_vocab(self, vocab: Vocab) -> None: + for text, score in vocab.all_tokens(): + self.fout.write(struct.pack("i", len(text))) + self.fout.write(text) + self.fout.write(struct.pack("f", score)) + + @staticmethod + def write_vocab_only(fname_out: Path, vocab: Vocab) -> None: + of = OutputFile(fname_out) + params = Params(n_vocab=vocab.vocab_size, n_embd=0, n_mult=0, n_head=1, n_layer=0) + of = OutputFile(fname_out) + of.write_file_header(params, file_type=GGMLFileType.AllF32) + of.write_vocab(vocab) + of.fout.close() + + @staticmethod + def write_all(fname_out: Path, params: Params, file_type: GGMLFileType, model: LazyModel, vocab: Vocab) -> None: + check_vocab_size(params, vocab) + of = OutputFile(fname_out) + of.write_file_header(params, file_type) + print("Writing vocab...") + of.write_vocab(vocab) + + def do_item(item: Tuple[str, LazyTensor]) -> NDArray: + name, lazy_tensor = item + return lazy_tensor.load().to_ggml().ndarray + + ndarrays = bounded_parallel_map(do_item, model.items(), concurrency=8) + for i, ((name, lazy_tensor), ndarray) in enumerate(zip(model.items(), ndarrays)): + size = ' x '.join(f"{dim:6d}" for dim in lazy_tensor.shape) + padi = len(str(len(model))) + print(f"[{i+1:{padi}d}/{len(model)}] Writing tensor {name:38s} | size {size:16} | type {lazy_tensor.data_type}") + of.write_tensor_header(name, lazy_tensor.shape, lazy_tensor.data_type) + ndarray.tofile(of.fout) + of.fout.close() + + +def pick_output_type(model: LazyModel, output_type_str: Optional[str]) -> GGMLFileType: + wq_type = model["layers.0.attention.wq.weight"].data_type + if output_type_str == "f32" or (output_type_str is None and wq_type in (DT_F32, DT_BF16)): + return GGMLFileType.AllF32 + if output_type_str == "f16" or (output_type_str is None and wq_type == DT_F16): + return GGMLFileType.MostlyF16 + name_to_type = {name: lazy_tensor.data_type for (name, lazy_tensor) in model.items()} + raise Exception(f"Unexpected combination of types: {name_to_type}") + + +def do_necessary_conversions(model: LazyModel, params: Params) -> LazyModel: + if "lm_head.weight" in model: + model = convert_transformers_to_orig(model, params) + model = filter_and_sort_tensors(model) + + return model + + +def convert_to_output_type(model: LazyModel, output_type: GGMLFileType) -> LazyModel: + return {name: tensor.astype(output_type.type_for_tensor(name, tensor)) + for (name, tensor) in model.items()} + + +def nth_multifile_path(path: Path, n: int) -> Optional[Path]: + '''Given any path belonging to a multi-file model (e.g. foo.bin.1), return + the nth path in the model. + ''' + # Support the following patterns: + patterns: List[Tuple[str, str]] = [ + # - x.00.pth, x.01.pth, etc. + (r'\.[0-9]{2}\.pth$', f'.{n:02}.pth'), + # - x-00001-of-00002.bin, x-00002-of-00002.bin, etc. + (r'-[0-9]{5}-of-(.*)$', fr'-{n:05}-of-\1'), + # x.bin, x.bin.1, etc. + (r'(\.[0-9]+)?$', r'\1' if n == 0 else fr'\1.{n}') + ] + for regex, replacement in patterns: + if re.search(regex, path.name): + new_path = path.with_name(re.sub(regex, replacement, path.name)) + if new_path.exists(): + return new_path + return None + + +def find_multifile_paths(path: Path) -> List[Path]: + '''Given any path belonging to a multi-file model (e.g. foo.bin.1), return + the whole list of paths in the model. + ''' + ret: List[Path] = [] + for i in itertools.count(): + nth_path = nth_multifile_path(path, i) + if nth_path is None: + break + ret.append(nth_path) + if not ret: + # No matches. This should only happen if the file was named, e.g., + # foo.0, and there was no file named foo. Oh well, try to process it + # as a single file. + return [path] + return ret + + +def load_some_model(path: Path) -> ModelPlus: + '''Load a model of any supported format.''' + # Be extra-friendly and accept either a file or a directory: + if path.is_dir(): + # Check if it's a set of safetensors files first + files = list(path.glob("model-00001-of-*.safetensors")) + if not files: + # Try the PyTorch patterns too, with lower priority + globs = ["consolidated.00.pth", "pytorch_model-00001-of-*.bin", "*.pt", "pytorch_model.bin"] + files = [file for glob in globs for file in path.glob(glob)] + if not files: + # Try GGML too, but with lower priority, since if both a non-GGML + # model and a GGML model exist in the same directory, we assume the + # latter was converted from the former. + files = list(path.glob("ggml-model*.bin*")) + if not files: + raise Exception(f"Can't find model in directory {path}") + if len(files) > 1: + raise Exception(f"Found multiple models in {path}, not sure which to pick: {files}") + path = files[0] + + paths = find_multifile_paths(path) + models_plus: List[ModelPlus] = [] + for path in paths: + print(f"Loading model file {path}") + models_plus.append(lazy_load_file(path)) + + model_plus = merge_multifile_models(models_plus) + return model_plus + + +def filter_and_sort_tensors(model: LazyModel) -> LazyModel: + return {name: model[name] for name in TENSORS_LIST if name in model} + + +def load_vocab(path: Path, vocabtype: Optional[str]) -> SentencePieceVocab: + print(f"vocabtype: {vocabtype}") + # Be extra-friendly and accept either a file or a directory. Also, if it's + # a directory, it might be the model directory, and tokenizer.model might + # be in the parent of that. + if path.is_dir(): + vocab_file = "tokenizer.model" + if vocabtype == 'bpe': + vocab_file = "vocab.json" + path2 = path / vocab_file + # Use `.parent` instead of /.. to handle the symlink case better. + path3 = path.parent / vocab_file + if path2.exists(): + path = path2 + elif path3.exists(): + path = path3 + else: + raise FileNotFoundError( + f"Could not find tokenizer.model in {path} or its parent; " + "if it's in another directory, pass the directory as --vocab-dir") + added_tokens_path = path.parent / "added_tokens.json" + print(f"Loading vocab file {path}") + return SentencePieceVocab(path, added_tokens_path if added_tokens_path.exists() else None, + vocabtype) + + +def default_outfile(model_paths: List[Path], file_type: GGMLFileType) -> Path: + namestr = { + GGMLFileType.AllF32: "f32", + GGMLFileType.MostlyF16: "f16", + }[file_type] + ret = model_paths[0].parent / f"ggml-model-{namestr}.bin" + if ret in model_paths: + sys.stderr.write( + f"Error: Default output path ({ret}) would overwrite the input. " + "Please explicitly specify a path using --outfile.\n") + sys.exit(1) + return ret + + +def do_dump_model(model_plus: ModelPlus) -> None: + print(f"model_plus.paths = {model_plus.paths!r}") + print(f"model_plus.format = {model_plus.format!r}") + print(f"model_plus.vocab = {model_plus.vocab!r}") + for name, lazy_tensor in model_plus.model.items(): + print(f"{name}: shape={lazy_tensor.shape} type={lazy_tensor.data_type}; {lazy_tensor.description}") + + +def main(args_in: Optional[List[str]] = None) -> None: + parser = argparse.ArgumentParser(description="Convert a LLaMa model to a GGML compatible file") + parser.add_argument("--dump", action="store_true", help="don't convert, just show what's in the model") + parser.add_argument("--dump-single", action="store_true", help="don't convert, just show what's in a single model file") + parser.add_argument("--vocab-only", action="store_true", help="extract only the vocab") + parser.add_argument("--outtype", choices=["f32", "f16", "q4_1", "q4_0"], help="output format (default: based on input)") + parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file") + parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input") + parser.add_argument("model", type=Path, + help="directory containing model file, or model file itself (*.pth, *.pt, *.bin)") + parser.add_argument("--vocabtype", default='spm', choices=["spm", "bpe"], help="vocab format (default: spm)") + args = parser.parse_args(args_in) + + vocab: Vocab + if args.dump_single: + model_plus = lazy_load_file(args.model) + do_dump_model(model_plus) + elif args.vocab_only: + vocab = load_vocab(args.vocab_dir or args.model, args.vocabtype) + assert args.outfile, "need --outfile if using --vocab-only" + outfile = args.outfile + OutputFile.write_vocab_only(outfile, vocab) + print(f"Wrote {outfile}") + else: + model_plus = load_some_model(args.model) + if args.dump: + do_dump_model(model_plus) + return + if model_plus.vocab is not None and args.vocab_dir is None: + vocab = model_plus.vocab + else: + vocab_dir = args.vocab_dir if args.vocab_dir else model_plus.paths[0].parent + vocab = load_vocab(vocab_dir, args.vocabtype) + params = Params.load(model_plus) + model = model_plus.model + model = do_necessary_conversions(model, params) + output_type = pick_output_type(model, args.outtype) + model = convert_to_output_type(model, output_type) + outfile = args.outfile or default_outfile(model_plus.paths, output_type) + OutputFile.write_all(outfile, params, output_type, model, vocab) + print(f"Wrote {outfile}") + + +if __name__ == '__main__': + main() From d2bb3ac10b025e9c2e475a0bc9527a7286b63810 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 27 Jul 2023 16:36:35 +0300 Subject: [PATCH 019/242] convert.py : remove GGML vocab + other obsolete stuff --- convert-new.py | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/convert-new.py b/convert-new.py index dacad693fea1c..bf85cb55177fa 100755 --- a/convert-new.py +++ b/convert-new.py @@ -278,19 +278,7 @@ def __repr__(self) -> str: return f"" -class GGMLVocab: - def __init__(self, tokens: List[Tuple[bytes, float]]): - self.tokens = tokens - self.vocab_size = len(tokens) - - def all_tokens(self) -> Iterable[Tuple[bytes, float]]: - return self.tokens - - def __repr__(self) -> str: - return f"" - - -Vocab = Union[SentencePieceVocab, GGMLVocab] +Vocab = Union[SentencePieceVocab] def permute(weights: NDArray, n_head: int) -> NDArray: @@ -691,7 +679,6 @@ def bounded_parallel_map(func: Callable[[In], Out], iterable: Iterable[In], conc def check_vocab_size(params: Params, vocab: Vocab) -> None: if params.n_vocab != vocab.vocab_size: - # GGMLVocab comes from the same file as the model so shouldn't mismatch: assert isinstance(vocab, SentencePieceVocab) if params.n_vocab == vocab.vocab_size_base: print("Ignoring added_tokens.json since model matches vocab size without it.") @@ -874,7 +861,7 @@ def load_vocab(path: Path, vocabtype: Optional[str]) -> SentencePieceVocab: if path.is_dir(): vocab_file = "tokenizer.model" if vocabtype == 'bpe': - vocab_file = "vocab.json" + vocab_file = "vocab.json" path2 = path / vocab_file # Use `.parent` instead of /.. to handle the symlink case better. path3 = path.parent / vocab_file @@ -916,15 +903,14 @@ def do_dump_model(model_plus: ModelPlus) -> None: def main(args_in: Optional[List[str]] = None) -> None: parser = argparse.ArgumentParser(description="Convert a LLaMa model to a GGML compatible file") - parser.add_argument("--dump", action="store_true", help="don't convert, just show what's in the model") - parser.add_argument("--dump-single", action="store_true", help="don't convert, just show what's in a single model file") - parser.add_argument("--vocab-only", action="store_true", help="extract only the vocab") - parser.add_argument("--outtype", choices=["f32", "f16", "q4_1", "q4_0"], help="output format (default: based on input)") - parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file") - parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input") - parser.add_argument("model", type=Path, - help="directory containing model file, or model file itself (*.pth, *.pt, *.bin)") - parser.add_argument("--vocabtype", default='spm', choices=["spm", "bpe"], help="vocab format (default: spm)") + parser.add_argument("--dump", action="store_true", help="don't convert, just show what's in the model") + parser.add_argument("--dump-single", action="store_true", help="don't convert, just show what's in a single model file") + parser.add_argument("--vocab-only", action="store_true", help="extract only the vocab") + parser.add_argument("--outtype", choices=["f32", "f16"], help="output format (default: based on input)") + parser.add_argument("--vocab-dir", type=Path, help="directory containing tokenizer.model, if separate from model file") + parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input") + parser.add_argument("model", type=Path, help="directory containing model file, or model file itself (*.pth, *.pt, *.bin)") + parser.add_argument("--vocabtype", choices=["spm", "bpe"], help="vocab format (default: spm)") args = parser.parse_args(args_in) vocab: Vocab @@ -947,12 +933,14 @@ def main(args_in: Optional[List[str]] = None) -> None: else: vocab_dir = args.vocab_dir if args.vocab_dir else model_plus.paths[0].parent vocab = load_vocab(vocab_dir, args.vocabtype) - params = Params.load(model_plus) - model = model_plus.model - model = do_necessary_conversions(model, params) + + params = Params.load(model_plus) + model = model_plus.model + model = do_necessary_conversions(model, params) output_type = pick_output_type(model, args.outtype) - model = convert_to_output_type(model, output_type) - outfile = args.outfile or default_outfile(model_plus.paths, output_type) + model = convert_to_output_type(model, output_type) + outfile = args.outfile or default_outfile(model_plus.paths, output_type) + OutputFile.write_all(outfile, params, output_type, model, vocab) print(f"Wrote {outfile}") From 11ef380c2a3a68289fa4fda0403b69ff0f59ddb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 28 Jul 2023 11:34:16 +0300 Subject: [PATCH 020/242] GGUF : write tensor (#2426) * WIP: Write tensor * GGUF : Support writing tensors in Python * refactor : rm unused import and upd todos * fix : fix errors upd writing example * rm example.gguf * gitignore *.gguf * undo formatting --- .gitignore | 1 + constants.py | 5 ++- gguf.py | 103 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 73 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index abe8e28cb7687..fd41f26cc0d20 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.o *.a *.so +*.gguf .DS_Store .build/ .cache/ diff --git a/constants.py b/constants.py index 3a97460e5dc5e..34880bb20db9c 100644 --- a/constants.py +++ b/constants.py @@ -1,5 +1,6 @@ -GGUF_MAGIC = 0x47475546 -GGUF_VERSION = 1 +GGUF_MAGIC = 0x47475546 +GGUF_VERSION = 1 +GGUF_DEFAULT_ALIGNMENT = 32 # general KEY_GENERAL_ARCHITECTURE = "general.architecture" diff --git a/gguf.py b/gguf.py index 764ae9a9df27c..c5b2174c9070a 100644 --- a/gguf.py +++ b/gguf.py @@ -1,14 +1,16 @@ """TODOs -1. Implement writing tensor data with alignment. -2. Implement writers for known architectures, LLaMA in particular. -3. Add docstrings from the format specs. -4. After development is done, Convert it to a proper pip-installable Python package, and possibly move it to its own repo under ggml-org. +1. Implement writers for known architectures, LLaMA in particular. +2. Add docstrings from the format specs. +3. After development is done, Convert it to a proper pip-installable Python package, and possibly move it to its own repo under ggml-org. """ import struct import constants from enum import IntEnum -from typing import List, Any +from typing import Any, IO, List + +import numpy as np + class GGMLQuantizationType(IntEnum): F32 = 0 @@ -54,15 +56,18 @@ def get_type(val): else: return GGUFValueType.INT32 + class GGUFWriter: - def __init__(self, buffered_writer): - self.buffered_writer = buffered_writer + def __init__(self, fout: IO): + self.fout = fout + self.offset_tensor = 0 + self.tensors: List[np.ndarray] = [] def write_header(self, tensor_count: int, metadata_kv_count: int): - self.buffered_writer.write(struct.pack(" "GGUFWriter": @@ -119,40 +124,69 @@ def write_val(self: str, val: Any, vtype: GGUFValueType = None): if vtype is None: vtype = GGUFValueType.get_type(val) - self.buffered_writer.write(struct.pack(" int: + return ((x + n - 1) // n) * n + + def write_tensor_info(self, name: str, tensor: np.ndarray): + self.write_val(name, GGUFValueType.STRING) + n_dims = len(tensor.shape) + self.write_val(n_dims, GGUFValueType.INT32) + for i in range(n_dims): + self.write_val(tensor.shape[n_dims - 1 - i], GGUFValueType.INT32) + + assert tensor.dtype in (np.float32, np.float16), "Only F32 and F16 tensors are supported for now" + dtype = GGMLQuantizationType.F32 if tensor.dtype == np.float32 else GGMLQuantizationType.F16 + self.write_val(dtype, GGUFValueType.INT32) + self.fout.write(struct.pack(" Date: Fri, 28 Jul 2023 22:45:24 +0200 Subject: [PATCH 021/242] gguf : add gguf_find_key (#2438) * gguf.cpp : find key example * ggml.h : add gguf_find_key * ggml.c : add gguf_find_key --- examples/gguf/gguf.cpp | 14 ++++++++++++++ ggml.c | 15 +++++++++++++++ ggml.h | 1 + 3 files changed, 30 insertions(+) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index c3494a343a6ec..35a870a8343f5 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -253,6 +253,20 @@ bool gguf_ex_read_0(const std::string & fname) { } } + // find kv string + { + char findkey[32]; + sprintf(findkey, "some.parameter.string"); + + int keyidx = gguf_find_key(ctx, findkey); + if (keyidx == -1) { + fprintf(stdout, "%s: find key: %s not found.\n", __func__, findkey); + } else { + const char * key_value = gguf_get_val_str(ctx, keyidx); + fprintf(stdout, "%s: find key: %s found, kv[%d] value = %s\n", __func__, findkey, keyidx, key_value); + } + } + // tensor info { const int n_tensors = gguf_get_n_tensors(ctx); diff --git a/ggml.c b/ggml.c index 96c7ebd347c1b..d402daa678b90 100644 --- a/ggml.c +++ b/ggml.c @@ -18745,6 +18745,21 @@ int gguf_get_n_kv(struct gguf_context * ctx) { return ctx->header.n_kv; } +int gguf_find_key(struct gguf_context * ctx, const char * key) { + // return -1 if key not found + const int n_kv = gguf_get_n_kv(ctx); + int keyfound = -1; + + for (int i = 0; i < n_kv; ++i) { + if (strcmp(key, gguf_get_key(ctx, i)) == 0) { + keyfound = i; + break; + } + } + + return keyfound; +} + const char * gguf_get_key(struct gguf_context * ctx, int i) { return ctx->header.kv[i].key.data; } diff --git a/ggml.h b/ggml.h index e857b3f14435f..a72c82a333cf5 100644 --- a/ggml.h +++ b/ggml.h @@ -1653,6 +1653,7 @@ extern "C" { GGML_API void * gguf_get_data (struct gguf_context * ctx); GGML_API int gguf_get_n_kv(struct gguf_context * ctx); + GGML_API int gguf_find_key(struct gguf_context * ctx, const char * key); GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); GGML_API void gguf_get_val (struct gguf_context * ctx, int i, void * val); From 1495735aaced6ca90209c7b3716ac77376b90d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 29 Jul 2023 00:26:22 +0300 Subject: [PATCH 022/242] gguf : fix writing tensors --- gguf.py | 56 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/gguf.py b/gguf.py index c5b2174c9070a..86c122a5e77d1 100644 --- a/gguf.py +++ b/gguf.py @@ -13,9 +13,9 @@ class GGMLQuantizationType(IntEnum): - F32 = 0 - F16 = 1 - QR_0 = 2 + F32 = 0 + F16 = 1 + Q4_0 = 2 Q4_1 = 3 # Q4_2 = 4 # support has been removed # Q4_3 = 5 # support has been removed @@ -32,16 +32,16 @@ class GGMLQuantizationType(IntEnum): class GGUFValueType(IntEnum): - UINT8 = 0 - INT8 = 1 - UINT16 = 2 - INT16 = 3 - UINT32 = 4 - INT32 = 5 + UINT8 = 0 + INT8 = 1 + UINT16 = 2 + INT16 = 3 + UINT32 = 4 + INT32 = 5 FLOAT32 = 6 - BOOL = 7 - STRING = 8 - ARRAY = 9 + BOOL = 7 + STRING = 8 + ARRAY = 9 @staticmethod def get_type(val): @@ -75,7 +75,9 @@ def open(cls, path: str) -> "GGUFWriter": return cls(f) def write_key(self, key: str): - self.write_val(key, GGUFValueType.STRING) + encoded_key = key.encode("utf8") + self.fout.write(struct.pack(" int: return ((x + n - 1) // n) * n def write_tensor_info(self, name: str, tensor: np.ndarray): - self.write_val(name, GGUFValueType.STRING) + self.write_key(name) n_dims = len(tensor.shape) - self.write_val(n_dims, GGUFValueType.INT32) + self.fout.write(struct.pack(" Date: Sat, 29 Jul 2023 10:24:46 +0300 Subject: [PATCH 023/242] gguf : do not hardcode tensor names to read --- examples/gguf/gguf.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index 35a870a8343f5..84388c219f5ef 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -296,7 +296,7 @@ bool gguf_ex_read_1(const std::string & fname) { }; struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params); - + fprintf(stdout, "%s: version: %d\n", __func__, gguf_get_version(ctx)); fprintf(stdout, "%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); fprintf(stdout, "%s: data offset: %zu\n", __func__, gguf_get_data_offset(ctx)); @@ -335,9 +335,9 @@ bool gguf_ex_read_1(const std::string & fname) { for (int i = 0; i < n_tensors; ++i) { fprintf(stdout, "%s: reading tensor %d data\n", __func__, i); - const std::string name = "tensor_" + to_string(i); + const char * name = gguf_get_tensor_name(ctx, i); - struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name.c_str()); + struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); fprintf(stdout, "%s: tensor[%d]: n_dims = %d, name = %s, data = %p\n", __func__, i, cur->n_dims, cur->name, cur->data); From 06f423a8e1ca6c4c1d348a5c6003bbab05644807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 29 Jul 2023 10:26:26 +0300 Subject: [PATCH 024/242] gguf : write sample tensors to read --- gguf.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/gguf.py b/gguf.py index 86c122a5e77d1..25d99ae93dbcd 100644 --- a/gguf.py +++ b/gguf.py @@ -179,14 +179,12 @@ def write_tensor_info(self, name: str, tensor: np.ndarray): def write_tensors(self): offset_data = GGUFWriter.ggml_pad(self.fout.tell(), constants.GGUF_DEFAULT_ALIGNMENT) pad = offset_data - self.fout.tell() - print(f"pad: {pad}") if pad != 0: self.fout.write(bytes([0] * pad)) for tensor in self.tensors: tensor.tofile(self.fout) pad = GGUFWriter.ggml_pad(tensor.nbytes, constants.GGUF_DEFAULT_ALIGNMENT) - tensor.nbytes - print(f"pad: {pad}") if pad != 0: self.fout.write(bytes([0] * pad)) @@ -282,10 +280,10 @@ def write_rope_scale(self, llm: str, value: float): gguf_writer.write_architecture("llama") gguf_writer.write_uint32("answer", 42) # Write a 32-bit integer gguf_writer.write_float32("answer_in_float", 42.0) # Write a 32-bit float - tensor1 = np.ones((7, 8, 3), dtype=np.float32) - tensor2 = np.ones((7, 8, 3), dtype=np.float32) - gguf_writer.write_tensor_info("tensor1", tensor1) - gguf_writer.write_tensor_info("tensor2", tensor2) + tensor1 = np.ones((32,), dtype=np.float32) * 100.0 + tensor2 = np.ones((32,), dtype=np.float32) * 101.0 + gguf_writer.write_tensor_info("tensor0", tensor1) + gguf_writer.write_tensor_info("tensor1", tensor2) gguf_writer.write_tensors() gguf_writer.close() From d54f53ca51a35af80bc6bf15849042c717587038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 29 Jul 2023 12:04:45 +0300 Subject: [PATCH 025/242] gguf : add tokenization constants --- constants.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/constants.py b/constants.py index 34880bb20db9c..024bf7b03bbe1 100644 --- a/constants.py +++ b/constants.py @@ -31,3 +31,16 @@ # RoPE KEY_ROPE_DIMENSION_COUNT = "{llm}.rope.dimension_count" KEY_ROPE_SCALE = "{llm}.rope.scale" + +# tokenization +KEY_TOKENIZER_MODEL = "tokenizer.ggml.model" +KEY_TOKENIZER_LIST = "tokenizer.ggml.tokens" +KEY_TOKENIZER_SCORES = "tokenizer.ggml.scores" +KEY_TOKENIZER_MERGES = "tokenizer.ggml.merges" +KEY_TOKENIZER_BOS_ID = "tokenizer.ggml.bos_token_id" +KEY_TOKENIZER_EOS_ID = "tokenizer.ggml.eos_token_id" +KEY_TOKENIZER_UNK_ID = "tokenizer.ggml.unknown_token_id" +KEY_TOKENIZER_SEP_ID = "tokenizer.ggml.seperator_token_id" +KEY_TOKENIZER_PAD_ID = "tokenizer.ggml.padding_token_id" +KEY_TOKENIZER_HF_JSON = "tokenizer.huggingface.json" +KEY_TOKENIZER_RWKV = "tokenizer.rwkv.world" From 999431c4b651ec4faa2b8715ce5f32ceeb3b9cb4 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sat, 29 Jul 2023 11:20:05 +0200 Subject: [PATCH 026/242] quick and dirty conversion example --- convert-llama-h5-to-gguf.py | 231 ++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 convert-llama-h5-to-gguf.py diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py new file mode 100644 index 0000000000000..29daed3b3cf48 --- /dev/null +++ b/convert-llama-h5-to-gguf.py @@ -0,0 +1,231 @@ +# Quick and dirty HF llama --> gguf conversion, GQA/70b wont work + +import gguf +import sys +import struct +import json +import numpy as np +from typing import List +from pathlib import Path +from transformers import AutoModelForCausalLM +from sentencepiece import SentencePieceProcessor + + +NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]' +def permute(weights: NDArray, n_head: int) -> NDArray: + return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) + .swapaxes(1, 2) + .reshape(weights.shape)) + +if len(sys.argv) < 3: + print("Usage: convert-h5-to-ggml.py dir-model ftype\n") + print(" ftype == 0 -> float32") + print(" ftype == 1 -> float16") + sys.exit(1) + + +# output in the same directory as the model +dir_model = sys.argv[1] +fname_out = sys.argv[1] + "/ggml-model.bin" + + +# possible tensor data types +# ftype == 0 -> float32 +# ftype == 1 -> float16 +# +# map from ftype to string +ftype_str = ["f32", "f16"] + +ftype = 1 +if len(sys.argv) > 2: + ftype = int(sys.argv[2]) + if ftype < 0 or ftype > 1: + print("Invalid ftype: " + str(ftype)) + sys.exit(1) + fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" + + +model = AutoModelForCausalLM.from_pretrained( dir_model, low_cpu_mem_usage=True, trust_remote_code=True ) +list_vars = model.state_dict() + +# count tensors to be converted +tensor_count = 0 +for name in list_vars.keys(): + # we don't need these + if name.endswith(".rotary_emb.inv_freq"): + continue + tensor_count += 1 + +#fout = open(fname_out, "wb") +gguf_writer = gguf.GGUFWriter.open(fname_out) + +with open(dir_model + "/config.json", "r", encoding="utf-8") as f: + hparams = json.load(f) + +# This mmust be changed when adding/deleting kv +kv_count = 13 + +print("tensors " + str(tensor_count) + " kv " + str(kv_count) ) + +print("write gguf header") + +gguf_writer.write_header(tensor_count, kv_count) + +print("write gguf hparams") + +llm_arch = "llama" + +gguf_writer.write_name("llama2-7b") +gguf_writer.write_description("gguf test model") +gguf_writer.write_architecture(llm_arch) +gguf_writer.write_context_length(llm_arch, hparams["max_position_embeddings"]) +gguf_writer.write_embedding_length(llm_arch, hparams["hidden_size"]) +gguf_writer.write_layer_count(llm_arch, hparams["num_hidden_layers"]) +gguf_writer.write_feed_forward_length(llm_arch, hparams["intermediate_size"]) +gguf_writer.write_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"]) +gguf_writer.write_head_count(llm_arch, hparams["num_attention_heads"]) +gguf_writer.write_float32(llm_arch + ".attention.layer_norm_rms_epsilon", hparams["rms_norm_eps"]) + + +# TOKENIZATION + +tokens: List[str] = [] +scores: List[float] = [] + +if Path( dir_model + "/tokenizer.model").is_file(): + # vocab type SPIECE + print( "Adding sentencepiece tokenizer vocab." ) + tokenizer = SentencePieceProcessor( dir_model + "/tokenizer.model" ) + + # output vocab_size followed by all piece/score pairs + outbytes: bytes + outbytes = b"" + outbytes += struct.pack("I", tokenizer.vocab_size()) + + for i in range(tokenizer.vocab_size()): + text: bytes + if tokenizer.is_unknown(i): + text = " \u2047 ".encode("utf-8") + elif tokenizer.is_control(i): + text = b"" + if tokenizer.is_byte(i): + piece = tokenizer.id_to_piece(i) + if len(piece) != 6: + raise Exception(f"Invalid token: {piece}") + byte_value = int(piece[3:-1], 16) + text = struct.pack("B", byte_value) + else: + text = tokenizer.id_to_piece(i).replace("\u2581", " ").encode("utf-8") + score: float = tokenizer.get_score(i) + + tokens.append( str(text) ); + scores.append( score ); + +print("write gguf tokens") + +gguf_writer.write_string("tokenizer.ggml.model", "llama") +gguf_writer.write_array("tokenizer.ggml.tokens",tokens) +gguf_writer.write_array("tokenizer.ggml.scores",scores) + +# TENSORS + + +# tensor info +print("write gguf tensor info") + +for name in list_vars.keys(): + data = list_vars[name].squeeze().numpy() + + # we don't need these + if name.endswith(".rotary_emb.inv_freq"): + continue + + # permute these + if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): + data = permute( data, hparams["num_attention_heads"] ) + + # chnage tensor name + + if name == "model.embed_tokens.weight": + name = "tok_embeddings.weight" + elif name == "model.norm.weight": + name = "norm.weight" + elif name == "lm_head.weight": + name = "output.weight" + else: + for i in range(80): # maximum number of layers + if name == "model.layers." + str(i) + ".input_layernorm.weight": + name = "layers." + str(i) + ".attention_norm.weight" + break + if name == "model.layers." + str(i) + ".self_attn.q_proj.weight": + name = "layers." + str(i) + ".attention.wq.weight" + break + if name == "model.layers." + str(i) + ".self_attn.k_proj.weight": + name = "layers." + str(i) + ".attention.wk.weight" + break + if name == "model.layers." + str(i) + ".self_attn.v_proj.weight": + name = "layers." + str(i) + ".attention.wv.weight" + break + if name == "model.layers." + str(i) + ".self_attn.o_proj.weight": + name = "layers." + str(i) + ".attention.wo.weight" + break + if name == "model.layers." + str(i) + ".post_attention_layernorm.weight": + name = "layers." + str(i) + ".ffn_norm.weight" + break + if name == "model.layers." + str(i) + ".mlp.gate_proj.weight": + name = "layers." + str(i) + ".feed_forward.w1.weight" + break + if name == "model.layers." + str(i) + ".mlp.down_proj.weight": + name = "layers." + str(i) + ".feed_forward.w2.weight" + break + if name == "model.layers." + str(i) + ".mlp.up_proj.weight": + name = "layers." + str(i) + ".feed_forward.w3.weight" + break + + gguf_writer.write_tensor_info(name, data) + + +# tensor data +print("write gguf tensor data") + +for name in list_vars.keys(): + data = list_vars[name].squeeze().numpy() + print("Process tensor: " + name + " with shape: ", data.shape) + + # we don't need these + if name.endswith(".rotary_emb.inv_freq"): + print(" Skip tensor: " + name) + continue + + ## permute these + if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): + print(" Permute tensor: " + name) + data = permute( data, hparams["num_attention_heads"] ) + + n_dims = len(data.shape) + + # ftype == 0 -> float32, ftype == 1 -> float16 + ftype_cur = 0 + if ftype != 0: + if name.endswith(".weight") and n_dims == 2: + print(" Converting to float16") + data = data.astype(np.float16) + ftype_cur = 1 + else: + print(" Converting to float32") + data = data.astype(np.float32) + ftype_cur = 0 + else: + if data.dtype != np.float32: + print(" Converting to float32") + data = data.astype(np.float32) + ftype_cur = 0 + + gguf_writer.write_tensor_padding() + gguf_writer.write_tensor(data) + +gguf_writer.close() + + +print("Done. Output file: " + fname_out) +print("") From ea5f9ad2ca4311dcad93f43bd020a88a72fc0cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 29 Jul 2023 12:25:43 +0300 Subject: [PATCH 027/242] gguf : fix writing gguf arrays --- gguf.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/gguf.py b/gguf.py index 25d99ae93dbcd..ac376606ffaf7 100644 --- a/gguf.py +++ b/gguf.py @@ -122,11 +122,12 @@ def write_array(self, key: str, val: list): self.write_key(key) self.write_val(val, GGUFValueType.ARRAY) - def write_val(self: str, val: Any, vtype: GGUFValueType = None): + def write_val(self: str, val: Any, vtype: GGUFValueType = None, write_vtype: bool = True): if vtype is None: vtype = GGUFValueType.get_type(val) - self.fout.write(struct.pack(" Date: Sat, 29 Jul 2023 12:34:35 +0300 Subject: [PATCH 028/242] gguf : write tensors one by one and code reuse --- gguf.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/gguf.py b/gguf.py index ac376606ffaf7..c5a8f5b909630 100644 --- a/gguf.py +++ b/gguf.py @@ -61,7 +61,6 @@ class GGUFWriter: def __init__(self, fout: IO): self.fout = fout self.offset_tensor = 0 - self.tensors: List[np.ndarray] = [] def write_header(self, tensor_count: int, metadata_kv_count: int): self.fout.write(struct.pack(" "GGUFWriter": return cls(f) def write_key(self, key: str): - encoded_key = key.encode("utf8") - self.fout.write(struct.pack(" Date: Sat, 29 Jul 2023 12:42:54 +0300 Subject: [PATCH 029/242] gguf : fix writing gguf arrays --- gguf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gguf.py b/gguf.py index c5a8f5b909630..de75b54d43137 100644 --- a/gguf.py +++ b/gguf.py @@ -147,9 +147,10 @@ def write_val(self: str, val: Any, vtype: GGUFValueType = None, write_vtype: boo self.fout.write(struct.pack(" Date: Sat, 29 Jul 2023 12:49:01 +0300 Subject: [PATCH 030/242] gguf : write tensors one by one --- gguf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gguf.py b/gguf.py index de75b54d43137..1888b6c2dec8f 100644 --- a/gguf.py +++ b/gguf.py @@ -283,6 +283,7 @@ def write_rope_scale(self, llm: str, value: float): tensor2 = np.ones((32,), dtype=np.float32) * 101.0 gguf_writer.write_tensor_info("tensor0", tensor1) gguf_writer.write_tensor_info("tensor1", tensor2) - gguf_writer.write_tensors() + gguf_writer.write_tensor(tensor1) + gguf_writer.write_tensor(tensor2) gguf_writer.close() From 8a76dd8a85bf2fd5c0c727c7915c923973d13564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 29 Jul 2023 13:17:28 +0300 Subject: [PATCH 031/242] gguf : write tensors one by one --- gguf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gguf.py b/gguf.py index 1888b6c2dec8f..cd727f3ea7a84 100644 --- a/gguf.py +++ b/gguf.py @@ -175,8 +175,6 @@ def write_tensor_info(self, name: str, tensor: np.ndarray): self.flush() - self.tensors.append(tensor) - def write_tensor(self, tensor: np.ndarray): pad = GGUFWriter.ggml_pad(self.fout.tell(), constants.GGUF_DEFAULT_ALIGNMENT) - self.fout.tell() if pad != 0: From cc3dd7f0420de545ca39b3f0ea6ca47273e5a439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 29 Jul 2023 13:30:22 +0300 Subject: [PATCH 032/242] gguf : write tokenizer data --- gguf.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gguf.py b/gguf.py index cd727f3ea7a84..e0b4b88fb11b8 100644 --- a/gguf.py +++ b/gguf.py @@ -267,6 +267,15 @@ def write_rope_dimension_count(self, llm: str, count: int): def write_rope_scale(self, llm: str, value: float): self.write_float32(constants.KEY_ROPE_SCALE.format(llm=llm), value) + def write_tokenizer_model(self, model: str): + self.write_string(constants.KEY_TOKENIZER_MODEL, model) + + def write_token_list(self, tokens: List[str]): + self.write_array(constants.KEY_TOKENIZER_LIST, tokens) + + def write_token_scores(self, scores: List[float]): + self.write_array(constants.KEY_TOKENIZER_SCORES, scores) + # Example usage: if __name__ == "__main__": From 0317c41d9877d6583b8469710fc0aefcf4ba2965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 29 Jul 2023 13:31:07 +0300 Subject: [PATCH 033/242] gguf : upd gguf conversion script --- convert-llama-h5-to-gguf.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 29daed3b3cf48..439a6da307f88 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -12,11 +12,14 @@ NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]' + + def permute(weights: NDArray, n_head: int) -> NDArray: return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) .swapaxes(1, 2) .reshape(weights.shape)) + if len(sys.argv) < 3: print("Usage: convert-h5-to-ggml.py dir-model ftype\n") print(" ftype == 0 -> float32") @@ -45,7 +48,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" -model = AutoModelForCausalLM.from_pretrained( dir_model, low_cpu_mem_usage=True, trust_remote_code=True ) +model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) list_vars = model.state_dict() # count tensors to be converted @@ -56,7 +59,6 @@ def permute(weights: NDArray, n_head: int) -> NDArray: continue tensor_count += 1 -#fout = open(fname_out, "wb") gguf_writer = gguf.GGUFWriter.open(fname_out) with open(dir_model + "/config.json", "r", encoding="utf-8") as f: @@ -65,7 +67,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # This mmust be changed when adding/deleting kv kv_count = 13 -print("tensors " + str(tensor_count) + " kv " + str(kv_count) ) +print("tensors " + str(tensor_count) + " kv " + str(kv_count)) print("write gguf header") @@ -92,10 +94,10 @@ def permute(weights: NDArray, n_head: int) -> NDArray: tokens: List[str] = [] scores: List[float] = [] -if Path( dir_model + "/tokenizer.model").is_file(): +if Path(dir_model + "/tokenizer.model").is_file(): # vocab type SPIECE - print( "Adding sentencepiece tokenizer vocab." ) - tokenizer = SentencePieceProcessor( dir_model + "/tokenizer.model" ) + print("Adding sentencepiece tokenizer vocab.") + tokenizer = SentencePieceProcessor(dir_model + "/tokenizer.model") # output vocab_size followed by all piece/score pairs outbytes: bytes @@ -118,14 +120,14 @@ def permute(weights: NDArray, n_head: int) -> NDArray: text = tokenizer.id_to_piece(i).replace("\u2581", " ").encode("utf-8") score: float = tokenizer.get_score(i) - tokens.append( str(text) ); - scores.append( score ); + tokens.append(str(text)) + scores.append(score) print("write gguf tokens") -gguf_writer.write_string("tokenizer.ggml.model", "llama") -gguf_writer.write_array("tokenizer.ggml.tokens",tokens) -gguf_writer.write_array("tokenizer.ggml.scores",scores) +gguf_writer.write_tokenizer_model("llama") +gguf_writer.write_token_list(tokens) +gguf_writer.write_token_scores(scores) # TENSORS @@ -142,7 +144,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # permute these if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): - data = permute( data, hparams["num_attention_heads"] ) + data = permute(data, hparams["num_attention_heads"]) # chnage tensor name @@ -197,10 +199,10 @@ def permute(weights: NDArray, n_head: int) -> NDArray: print(" Skip tensor: " + name) continue - ## permute these + # permute these if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): print(" Permute tensor: " + name) - data = permute( data, hparams["num_attention_heads"] ) + data = permute(data, hparams["num_attention_heads"]) n_dims = len(data.shape) @@ -221,7 +223,6 @@ def permute(weights: NDArray, n_head: int) -> NDArray: data = data.astype(np.float32) ftype_cur = 0 - gguf_writer.write_tensor_padding() gguf_writer.write_tensor(data) gguf_writer.close() From 8ad7cd49fb0979aeafa39e42ce5f335a4ed53e35 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sat, 29 Jul 2023 16:47:00 +0200 Subject: [PATCH 034/242] Update convert-llama-h5-to-gguf.py --- convert-llama-h5-to-gguf.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 439a6da307f88..d36e6da9a30ce 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -120,7 +120,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: text = tokenizer.id_to_piece(i).replace("\u2581", " ").encode("utf-8") score: float = tokenizer.get_score(i) - tokens.append(str(text)) + tokens.append(text) scores.append(score) print("write gguf tokens") @@ -184,6 +184,22 @@ def permute(weights: NDArray, n_head: int) -> NDArray: name = "layers." + str(i) + ".feed_forward.w3.weight" break + n_dims = len(data.shape) + + # ftype == 0 -> float32, ftype == 1 -> float16 + ftype_cur = 0 + if ftype != 0: + if name.endswith(".weight") and n_dims == 2: + data = data.astype(np.float16) + ftype_cur = 1 + else: + data = data.astype(np.float32) + ftype_cur = 0 + else: + if data.dtype != np.float32: + data = data.astype(np.float32) + ftype_cur = 0 + gguf_writer.write_tensor_info(name, data) From 0f5e57f01d8913457f954e9b4d1ff9b9471b375a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 29 Jul 2023 19:56:06 +0300 Subject: [PATCH 035/242] gguf : handle already encoded string --- gguf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gguf.py b/gguf.py index e0b4b88fb11b8..88d7e43fb7fd3 100644 --- a/gguf.py +++ b/gguf.py @@ -45,7 +45,7 @@ class GGUFValueType(IntEnum): @staticmethod def get_type(val): - if isinstance(val, str): + if isinstance(val, str) or isinstance(val, bytes): return GGUFValueType.STRING elif isinstance(val, list): return GGUFValueType.ARRAY @@ -143,7 +143,7 @@ def write_val(self: str, val: Any, vtype: GGUFValueType = None, write_vtype: boo elif vtype == GGUFValueType.BOOL: self.fout.write(struct.pack("?", val)) elif vtype == GGUFValueType.STRING: - encoded_val = val.encode("utf8") + encoded_val = val.encode("utf8") if isinstance(val, str) else val self.fout.write(struct.pack(" Date: Sat, 29 Jul 2023 20:36:06 +0200 Subject: [PATCH 036/242] ggml.h : get array str and f32 --- ggml.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ggml.h b/ggml.h index a72c82a333cf5..0296508517e8e 100644 --- a/ggml.h +++ b/ggml.h @@ -1657,6 +1657,9 @@ extern "C" { GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); GGML_API void gguf_get_val (struct gguf_context * ctx, int i, void * val); + GGML_API const char * gguf_get_arr_str(struct gguf_context * ctx, int key_id, int i); + GGML_API float gguf_get_arr_f32(struct gguf_context * ctx, int key_id, int i); + GGML_API uint8_t gguf_get_val_u8 (struct gguf_context * ctx, int i); GGML_API int8_t gguf_get_val_i8 (struct gguf_context * ctx, int i); GGML_API uint16_t gguf_get_val_u16 (struct gguf_context * ctx, int i); From 2c22e3bcdb25d718761a2c2e69ef1dd894f747fa Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sat, 29 Jul 2023 20:37:47 +0200 Subject: [PATCH 037/242] ggml.c : get arr str and f32 --- ggml.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ggml.c b/ggml.c index d402daa678b90..af3a9ce71c664 100644 --- a/ggml.c +++ b/ggml.c @@ -18768,6 +18768,16 @@ enum gguf_type gguf_get_type(struct gguf_context * ctx, int i) { return ctx->header.kv[i].type; } +const char * gguf_get_arr_str(struct gguf_context * ctx, int key_id, int i) { + struct gguf_kv * kv = &ctx->header.kv[key_id]; + struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[i]; + return str->data; +} + +float gguf_get_arr_f32(struct gguf_context * ctx, int key_id, int i) { + return ((float *) ctx->header.kv[key_id].value.arr.data)[i]; +} + uint8_t gguf_get_val_u8(struct gguf_context * ctx, int i) { return ctx->header.kv[i].value.uint8; } From 9577821487909aac1ee33654ac0e9ade9a63f886 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sat, 29 Jul 2023 21:29:07 +0200 Subject: [PATCH 038/242] gguf.py : support any type --- gguf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gguf.py b/gguf.py index 88d7e43fb7fd3..7e15e5a777d16 100644 --- a/gguf.py +++ b/gguf.py @@ -270,10 +270,10 @@ def write_rope_scale(self, llm: str, value: float): def write_tokenizer_model(self, model: str): self.write_string(constants.KEY_TOKENIZER_MODEL, model) - def write_token_list(self, tokens: List[str]): + def write_token_list(self, tokens: List): self.write_array(constants.KEY_TOKENIZER_LIST, tokens) - def write_token_scores(self, scores: List[float]): + def write_token_scores(self, scores: List: self.write_array(constants.KEY_TOKENIZER_SCORES, scores) From 06c3e4a1a7eec3db5ef69c9f31dfa55b22c4e778 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sat, 29 Jul 2023 21:38:01 +0200 Subject: [PATCH 039/242] Update convert-llama-h5-to-gguf.py --- convert-llama-h5-to-gguf.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index d36e6da9a30ce..ba9e9f677305d 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -91,11 +91,13 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # TOKENIZATION +print("write gguf tokenizer") + tokens: List[str] = [] scores: List[float] = [] if Path(dir_model + "/tokenizer.model").is_file(): - # vocab type SPIECE + # vocab type sentencepiece print("Adding sentencepiece tokenizer vocab.") tokenizer = SentencePieceProcessor(dir_model + "/tokenizer.model") @@ -123,15 +125,12 @@ def permute(weights: NDArray, n_head: int) -> NDArray: tokens.append(text) scores.append(score) -print("write gguf tokens") - gguf_writer.write_tokenizer_model("llama") gguf_writer.write_token_list(tokens) gguf_writer.write_token_scores(scores) # TENSORS - # tensor info print("write gguf tensor info") From 32e037ffbe7af7be1bb620b6fe64861fe6de2843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sun, 30 Jul 2023 01:01:13 +0300 Subject: [PATCH 040/242] gguf : fix set is not subscriptable --- gguf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gguf.py b/gguf.py index 7e15e5a777d16..3bcc15f8bc755 100644 --- a/gguf.py +++ b/gguf.py @@ -149,7 +149,7 @@ def write_val(self: str, val: Any, vtype: GGUFValueType = None, write_vtype: boo elif vtype == GGUFValueType.ARRAY: ltype = set([GGUFValueType.get_type(item) for item in val]) assert len(ltype) == 1, "All items in a GGUF array should be of the same type" - self.fout.write(struct.pack(" Date: Sun, 30 Jul 2023 01:09:22 +0300 Subject: [PATCH 041/242] gguf : update convert-llama-h5-to-gguf.py --- convert-llama-h5-to-gguf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index ba9e9f677305d..0ea5e21aa0002 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -5,13 +5,13 @@ import struct import json import numpy as np -from typing import List +from typing import Any, List from pathlib import Path from transformers import AutoModelForCausalLM from sentencepiece import SentencePieceProcessor -NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]' +NDArray = np.ndarray[Any, Any] def permute(weights: NDArray, n_head: int) -> NDArray: From 0790c121aad79e35e89413d5031818413944d6d4 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 14:46:36 +0200 Subject: [PATCH 042/242] constants.py : add layer norm eps --- constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/constants.py b/constants.py index 024bf7b03bbe1..14f11123b1280 100644 --- a/constants.py +++ b/constants.py @@ -27,6 +27,8 @@ KEY_ATTENTION_HEAD_COUNT_KV = "{llm}.attention.head_count_kv" KEY_ATTENTION_MAX_ALIBI_BIAS = "{llm}.attention.max_alibi_bias" KEY_ATTENTION_CLAMP_KQV = "{llm}.attention.clamp_kqv" +KEY_ATTENTION_LAYERNORM_EPS = "{llm}.attention.layer_norm_epsilon" +KEY_ATTENTION_LAYERNORM_RMS_EPS = "{llm}.attention.layer_norm_rms_epsilon" # RoPE KEY_ROPE_DIMENSION_COUNT = "{llm}.rope.dimension_count" From ccd81a751bfd6f313d5bea7ea20cd2eee3ee53b0 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 14:48:14 +0200 Subject: [PATCH 043/242] gguf.py : add layer norm eps and merges --- gguf.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/gguf.py b/gguf.py index 3bcc15f8bc755..99ea817028c70 100644 --- a/gguf.py +++ b/gguf.py @@ -260,6 +260,14 @@ def write_clamp_kqv(self, llm: str, value: float): self.write_float32( constants.KEY_ATTENTION_CLAMP_KQV.format(llm=llm), value) + def write_layer_norm_eps(self, llm: str, value: float): + self.write_float32( + constants.KEY_ATTENTION_LAYERNORM_EPS.format(llm=llm), value) + + def write_layer_norm_rms_eps(self, llm: str, value: float): + self.write_float32( + constants.KEY_ATTENTION_LAYERNORM_RMS_EPS.format(llm=llm), value) + def write_rope_dimension_count(self, llm: str, count: int): self.write_uint32( constants.KEY_ROPE_DIMENSION_COUNT.format(llm=llm), count) @@ -273,6 +281,9 @@ def write_tokenizer_model(self, model: str): def write_token_list(self, tokens: List): self.write_array(constants.KEY_TOKENIZER_LIST, tokens) + def write_token_merges(self, merges: List): + self.write_array(constants.KEY_TOKENIZER_MERGES, merges) + def write_token_scores(self, scores: List[float]): self.write_array(constants.KEY_TOKENIZER_SCORES, scores) From b4676ee4479985741b17d9f0323d8d229fff285e Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 14:51:37 +0200 Subject: [PATCH 044/242] ggml.h : increase GGML_MAX_NAME to 64 --- ggml.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml.h b/ggml.h index 0296508517e8e..f939c30672387 100644 --- a/ggml.h +++ b/ggml.h @@ -198,7 +198,7 @@ #define GGML_MAX_PARAMS 256 #define GGML_MAX_CONTEXTS 64 #define GGML_MAX_SRC 6 -#define GGML_MAX_NAME 48 +#define GGML_MAX_NAME 64 #define GGML_MAX_OP_PARAMS 32 #define GGML_DEFAULT_N_THREADS 4 From b19c11750b20877561ddbec93a3c1f3bdd655d08 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 14:58:50 +0200 Subject: [PATCH 045/242] ggml.c : add gguf_get_arr_n --- ggml.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml.c b/ggml.c index af3a9ce71c664..1b2f071e71a3d 100644 --- a/ggml.c +++ b/ggml.c @@ -18778,6 +18778,10 @@ float gguf_get_arr_f32(struct gguf_context * ctx, int key_id, int i) { return ((float *) ctx->header.kv[key_id].value.arr.data)[i]; } +int gguf_get_arr_n(struct gguf_context * ctx, int i) { + return ctx->header.kv[i].value.arr.n; +} + uint8_t gguf_get_val_u8(struct gguf_context * ctx, int i) { return ctx->header.kv[i].value.uint8; } From 4ed98bf1ab5766a844df8bca92879ee793b7cf50 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 15:01:47 +0200 Subject: [PATCH 046/242] Update convert-llama-h5-to-gguf.py --- convert-llama-h5-to-gguf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 0ea5e21aa0002..926dd265b4a4c 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -87,6 +87,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: gguf_writer.write_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"]) gguf_writer.write_head_count(llm_arch, hparams["num_attention_heads"]) gguf_writer.write_float32(llm_arch + ".attention.layer_norm_rms_epsilon", hparams["rms_norm_eps"]) +gguf_writer.write_layer_norm_rms_eps(llm_arch, hparams["rms_norm_eps"]) # TOKENIZATION From e9192b0135c02201bc03702c16f12b27a5ab580d Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 15:05:37 +0200 Subject: [PATCH 047/242] add gptneox gguf example --- convert-gptneox-h5-to-gguf.py | 173 ++++++++++ gptneox-common.cpp | 601 ++++++++++++++++++++++++++++++++++ gptneox-common.h | 125 +++++++ 3 files changed, 899 insertions(+) create mode 100644 convert-gptneox-h5-to-gguf.py create mode 100644 gptneox-common.cpp create mode 100644 gptneox-common.h diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py new file mode 100644 index 0000000000000..cbade7079be8b --- /dev/null +++ b/convert-gptneox-h5-to-gguf.py @@ -0,0 +1,173 @@ +# Quick and dirty HF gptneox--> gguf conversion + +import gguf +import sys +import struct +import json +import numpy as np +from typing import Any, List +from pathlib import Path +from transformers import AutoModelForCausalLM + + +if len(sys.argv) < 3: + print("Usage: convert-h5-to-ggml.py dir-model ftype\n") + print(" ftype == 0 -> float32") + print(" ftype == 1 -> float16") + sys.exit(1) + + +# output in the same directory as the model +dir_model = sys.argv[1] +fname_out = sys.argv[1] + "/ggml-model.bin" + + +# possible tensor data types +# ftype == 0 -> float32 +# ftype == 1 -> float16 +# +# map from ftype to string +ftype_str = ["f32", "f16"] + +ftype = 1 +if len(sys.argv) > 2: + ftype = int(sys.argv[2]) + if ftype < 0 or ftype > 1: + print("Invalid ftype: " + str(ftype)) + sys.exit(1) + fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" + + +model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) +list_vars = model.state_dict() + +# count tensors to be converted +tensor_count = 0 +for name in list_vars.keys(): + # we don't need these + if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): + continue + tensor_count += 1 + +gguf_writer = gguf.GGUFWriter.open(fname_out) + +with open(dir_model + "/config.json", "r", encoding="utf-8") as f: + hparams = json.load(f) + +# This mmust be changed when adding/deleting kv +kv_count = 14 + +print("tensors " + str(tensor_count) + " kv " + str(kv_count)) + +print("write gguf header") + +gguf_writer.write_header(tensor_count, kv_count) + +print("write gguf hparams") + +llm_arch = "gptneox" + +gguf_writer.write_name("pythia-70b-deduped") +gguf_writer.write_description("gguf test model") +gguf_writer.write_architecture(llm_arch) +gguf_writer.write_context_length(llm_arch, hparams["max_position_embeddings"]) +gguf_writer.write_embedding_length(llm_arch, hparams["hidden_size"]) +gguf_writer.write_layer_count(llm_arch, hparams["num_hidden_layers"]) +gguf_writer.write_feed_forward_length(llm_arch, hparams["intermediate_size"]) +gguf_writer.write_rope_dimension_count(llm_arch, int( hparams["rotary_pct"]*(hparams["hidden_size"]//hparams["num_attention_heads"])) ) +gguf_writer.write_head_count(llm_arch, hparams["num_attention_heads"]) +gguf_writer.write_parallel_residual(llm_arch, hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True) +gguf_writer.write_layer_norm_eps(llm_arch, hparams["layer_norm_eps"]) + +# TOKENIZATION + +print("write gguf tokenizer") + +tokens: List[str] = [] +merges: List[str] = [] + +if Path(dir_model + "/tokenizer.json").is_file(): + # vocab type gpt2 + print("Adding gpt2 tokenizer vocab") + + with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f: + tokenizer = json.load(f) + + for key in tokenizer["model"]["vocab"]: + tokens.append(key) + + merges = tokenizer["model"]["merges"] + +gguf_writer.write_tokenizer_model("gpt2") +gguf_writer.write_token_list(tokens) +gguf_writer.write_token_merges(merges) + +# TENSORS + +# tensor info +print("write gguf tensor info") + +for name in list_vars.keys(): + data = list_vars[name].squeeze().numpy() + + # we don't need these + if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): + continue + + n_dims = len(data.shape) + + # ftype == 0 -> float32, ftype == 1 -> float16 + ftype_cur = 0 + if ftype != 0: + if name.endswith(".weight") and n_dims == 2: + data = data.astype(np.float16) + ftype_cur = 1 + else: + data = data.astype(np.float32) + ftype_cur = 0 + else: + if data.dtype != np.float32: + data = data.astype(np.float32) + ftype_cur = 0 + + gguf_writer.write_tensor_info(name, data) + + +# tensor data +print("write gguf tensor data") + +for name in list_vars.keys(): + data = list_vars[name].squeeze().numpy() + print("Process tensor: " + name + " with shape: ", data.shape) + + # we don't need these + if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): + print(" Skip tensor: " + name) + continue + + n_dims = len(data.shape) + + # ftype == 0 -> float32, ftype == 1 -> float16 + ftype_cur = 0 + if ftype != 0: + if name.endswith(".weight") and n_dims == 2: + print(" Converting to float16") + data = data.astype(np.float16) + ftype_cur = 1 + else: + print(" Converting to float32") + data = data.astype(np.float32) + ftype_cur = 0 + else: + if data.dtype != np.float32: + print(" Converting to float32") + data = data.astype(np.float32) + ftype_cur = 0 + + gguf_writer.write_tensor(data) + +gguf_writer.close() + + +print("Done. Output file: " + fname_out) +print("") diff --git a/gptneox-common.cpp b/gptneox-common.cpp new file mode 100644 index 0000000000000..9dee0cb9ce8b2 --- /dev/null +++ b/gptneox-common.cpp @@ -0,0 +1,601 @@ +#include "gptneox-common.h" + +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +// Function to check if the next argument exists +std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) { + if (i + 1 < argc && argv[i + 1][0] != '-') { + return argv[++i]; + } else { + fprintf(stderr, "error: %s requires one argument.\n", flag.c_str()); + gpt_print_usage(argc, argv, params); + exit(0); + } +} + +bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { + for (int i = 1; i < argc; i++) { + std::string arg = argv[i]; + + if (arg == "-s" || arg == "--seed") { + params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-t" || arg == "--threads") { + params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") { + params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-p" || arg == "--prompt") { + params.prompt = get_next_arg(i, argc, argv, arg, params); + } else if (arg == "-n" || arg == "--n_predict") { + params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--top_k") { + params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--top_p") { + params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--temp") { + params.temp = std::stof(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--repeat-last-n") { + params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--repeat-penalty") { + params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-b" || arg == "--batch_size") { + params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-m" || arg == "--model") { + params.model = get_next_arg(i, argc, argv, arg, params); + } else if (arg == "-i" || arg == "--interactive") { + params.interactive = true; + } else if (arg == "-ip" || arg == "--interactive-port") { + params.interactive = true; + params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-h" || arg == "--help") { + gpt_print_usage(argc, argv, params); + exit(0); + } else if (arg == "-f" || arg == "--file") { + get_next_arg(i, argc, argv, arg, params); + std::ifstream file(argv[i]); + if (!file) { + fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); + break; + } + std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.prompt)); + if (params.prompt.back() == '\n') { + params.prompt.pop_back(); + } + } else if (arg == "-tt" || arg == "--token_test") { + params.token_test = get_next_arg(i, argc, argv, arg, params); + } + else { + fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); + gpt_print_usage(argc, argv, params); + exit(0); + } + } + + return true; +} + +void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { + fprintf(stderr, "usage: %s [options]\n", argv[0]); + fprintf(stderr, "\n"); + fprintf(stderr, "options:\n"); + fprintf(stderr, " -h, --help show this help message and exit\n"); + fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n"); + fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); + fprintf(stderr, " -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers); + fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); + fprintf(stderr, " prompt to start generation with (default: random)\n"); + fprintf(stderr, " -f FNAME, --file FNAME\n"); + fprintf(stderr, " load prompt from a file\n"); + fprintf(stderr, " -tt TOKEN_TEST, --token_test TOKEN_TEST\n"); + fprintf(stderr, " test tokenization\n"); + fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d)\n", params.n_predict); + fprintf(stderr, " --top_k N top-k sampling (default: %d)\n", params.top_k); + fprintf(stderr, " --top_p N top-p sampling (default: %.1f)\n", params.top_p); + fprintf(stderr, " --temp N temperature (default: %.1f)\n", params.temp); + fprintf(stderr, " --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled)\n", params.repeat_last_n); + fprintf(stderr, " --repeat-penalty N penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\n", (double)params.repeat_penalty); + fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch); + fprintf(stderr, " -m FNAME, --model FNAME\n"); + fprintf(stderr, " model path (default: %s)\n", params.model.c_str()); + fprintf(stderr, "\n"); +} + +std::string gpt_random_prompt(std::mt19937 & rng) { + const int r = rng() % 10; + switch (r) { + case 0: return "So"; + case 1: return "Once upon a time"; + case 2: return "When"; + case 3: return "The"; + case 4: return "After"; + case 5: return "If"; + case 6: return "import"; + case 7: return "He"; + case 8: return "She"; + case 9: return "They"; + default: return "To"; + } + + return "The"; +} + +std::string trim(const std::string & s) { + std::regex e("^\\s+|\\s+$"); + return std::regex_replace(s, e, ""); +} + +std::string replace(const std::string & s, const std::string & from, const std::string & to) { + std::string result = s; + size_t pos = 0; + while ((pos = result.find(from, pos)) != std::string::npos) { + result.replace(pos, from.length(), to); + pos += to.length(); + } + return result; +} + +void gpt_vocab::add_special_token(const std::string & token) { + special_tokens.push_back(token); +} + +std::map json_parse(const std::string & fname) { + std::map result; + + // read file into string + std::string json; + { + std::ifstream ifs(fname); + if (!ifs) { + fprintf(stderr, "Failed to open %s\n", fname.c_str()); + exit(1); + } + + json = std::string((std::istreambuf_iterator(ifs)), + (std::istreambuf_iterator())); + } + + if (json[0] != '{') { + return result; + } + + // parse json + { + bool has_key = false; + bool in_token = false; + + std::string str_key = ""; + std::string str_val = ""; + + int n = json.size(); + for (int i = 1; i < n; ++i) { + if (!in_token) { + if (json[i] == ' ') continue; + if (json[i] == '"') { + in_token = true; + continue; + } + } else { + if (json[i] == '\\' && i+1 < n) { + if (has_key == false) { + str_key += json[i]; + } else { + str_val += json[i]; + } + ++i; + } else if (json[i] == '"') { + if (has_key == false) { + has_key = true; + ++i; + while (json[i] == ' ') ++i; + ++i; // : + while (json[i] == ' ') ++i; + if (json[i] != '\"') { + while (json[i] != ',' && json[i] != '}') { + str_val += json[i++]; + } + has_key = false; + } else { + in_token = true; + continue; + } + } else { + has_key = false; + } + + str_key = ::replace(str_key, "\\u0120", " " ); // \u0120 -> space + str_key = ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line + str_key = ::replace(str_key, "\\\"", "\""); // \\\" -> " + + try { + result[str_key] = std::stoi(str_val); + } catch (...) { + //fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str()); + + } + str_key = ""; + str_val = ""; + in_token = false; + continue; + } + if (has_key == false) { + str_key += json[i]; + } else { + str_val += json[i]; + } + } + } + } + + return result; +} + +std::string convert_to_utf8(const std::wstring & input) { + std::wstring_convert> converter; + return converter.to_bytes(input); +} + + +std::wstring convert_to_wstring(const std::string & input) { + std::wstring_convert> converter; + return converter.from_bytes(input); +} + +void gpt_split_words(std::string str, std::vector& words) { + const std::string pattern = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"; + const std::regex re(pattern); + std::smatch m; + + while (std::regex_search(str, m, re)) { + for (auto x : m) { + words.push_back(x); + } + str = m.suffix(); + } +} + +std::vector gpt_tokenize(const gpt_vocab & vocab, const std::string & text) { + std::vector words; + + // first split the text into words + { + std::string str = text; + + // Generate the subpattern from the special_tokens vector if it's not empty + if (!vocab.special_tokens.empty()) { + const std::regex escape(R"([\[\\\^\$\.\|\?\*\+\(\)\{\}])"); + std::string special_tokens_subpattern; + for (const auto & token : vocab.special_tokens) { + if (!special_tokens_subpattern.empty()) { + special_tokens_subpattern += "|"; + } + special_tokens_subpattern += std::regex_replace(token, escape, R"(\$&)"); + } + + std::regex re(special_tokens_subpattern); + std::smatch m; + // Split the text by special tokens. + while (std::regex_search(str, m, re)) { + // Split the substrings in-between special tokens into words. + gpt_split_words(m.prefix(), words); + // Add matched special tokens as words. + for (auto x : m) { + words.push_back(x); + } + str = m.suffix(); + } + // Remaining text without special tokens will be handled below. + } + + gpt_split_words(str, words); + } + + // find the longest token that forms each word in words: + std::vector tokens; + for (const auto & word : words) { + for (int i = 0; i < (int) word.size(); ){ + for (int j = word.size() - 1; j >= i; j--){ + auto cand = word.substr(i, j-i+1); + auto it = vocab.token_to_id.find(cand); + if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab + tokens.push_back(it->second); + i = j + 1; + break; + } + else if (j == i){ // word.substr(i, 1) has no matching + fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data()); + i++; + } + } + } + } + + return tokens; +} + +std::vector parse_tokens_from_string(const std::string& input, char delimiter) { + std::vector output; + std::stringstream ss(input); + std::string token; + + while (std::getline(ss, token, delimiter)) { + output.push_back(std::stoi(token)); + } + + return output; +} + +std::map> extract_tests_from_file(const std::string & fpath_test){ + if (fpath_test.empty()){ + fprintf(stderr, "%s : No test file found.\n", __func__); + return std::map>(); + } + + std::map> tests; + + auto fin = std::ifstream(fpath_test, std::ios_base::in); + const char * delimeter = " => "; + const char del_tok = ','; + std::string line; + while (std::getline(fin, line)) { + size_t delimiterPos = line.find(delimeter); + if (delimiterPos != std::string::npos) { + std::string text = line.substr(0, delimiterPos); + std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter)); + tests[text] = parse_tokens_from_string(s_tokens, del_tok); + } + } + return tests; +} + +void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){ + std::map> tests = extract_tests_from_file(fpath_test); + + size_t n_fails = 0; + + for (const auto & test : tests) { + std::vector tokens = gpt_tokenize(vocab, test.first); + + if (tokens != test.second){ + n_fails++; + + // print out failure cases + fprintf(stderr, "%s : failed test: '%s'\n", __func__, test.first.c_str()); + fprintf(stderr, "%s : tokens in hf: ", __func__); + for (const auto & t : test.second) { + fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t); + } + fprintf(stderr, "\n"); + fprintf(stderr, "%s : tokens in ggml: ", __func__); + for (const auto & t : tokens) { + fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t); + } + fprintf(stderr, "\n"); + } + } + + fprintf(stderr, "%s : %zu tests failed out of %zu tests.\n", __func__, n_fails, tests.size()); +} + +bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) { + printf("%s: loading vocab from '%s'\n", __func__, fname.c_str()); + + vocab.token_to_id = ::json_parse(fname); + + for (const auto & kv : vocab.token_to_id) { + vocab.id_to_token[kv.second] = kv.first; + } + + printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size()); + + // print the vocabulary + //for (auto kv : vocab.token_to_id) { + // printf("'%s' -> %d\n", kv.first.data(), kv.second); + //} + + return true; +} + +gpt_vocab::id gpt_sample_top_k_top_p( + const gpt_vocab & vocab, + const float * logits, + int top_k, + double top_p, + double temp, + std::mt19937 & rng) { + int n_logits = vocab.id_to_token.size(); + + std::vector> logits_id; + logits_id.reserve(n_logits); + + { + const double scale = 1.0/temp; + for (int i = 0; i < n_logits; ++i) { + logits_id.push_back(std::make_pair(logits[i]*scale, i)); + } + } + + // find the top K tokens + std::partial_sort( + logits_id.begin(), + logits_id.begin() + top_k, logits_id.end(), + [](const std::pair & a, const std::pair & b) { + return a.first > b.first; + }); + + logits_id.resize(top_k); + + double maxl = -INFINITY; + for (const auto & kv : logits_id) { + maxl = std::max(maxl, kv.first); + } + + // compute probs for the top K tokens + std::vector probs; + probs.reserve(logits_id.size()); + + double sum = 0.0; + for (const auto & kv : logits_id) { + double p = exp(kv.first - maxl); + probs.push_back(p); + sum += p; + } + + // normalize the probs + for (auto & p : probs) { + p /= sum; + } + + if (top_p < 1.0f) { + double cumsum = 0.0f; + for (int i = 0; i < top_k; i++) { + cumsum += probs[i]; + if (cumsum >= top_p) { + top_k = i + 1; + probs.resize(top_k); + logits_id.resize(top_k); + break; + } + } + + cumsum = 1.0/cumsum; + for (int i = 0; i < (int) probs.size(); i++) { + probs[i] *= cumsum; + } + } + + //printf("\n"); + //for (int i = 0; i < (int) probs.size(); i++) { + // printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]); + //} + //exit(0); + + std::discrete_distribution<> dist(probs.begin(), probs.end()); + int idx = dist(rng); + + return logits_id[idx].second; +} + +gpt_vocab::id gpt_sample_top_k_top_p_repeat( + const gpt_vocab & vocab, + const float * logits, + const int32_t * last_n_tokens_data, + size_t last_n_tokens_data_size, + int top_k, + double top_p, + double temp, + int repeat_last_n, + float repeat_penalty, + std::mt19937 & rng) { + + int n_logits = vocab.id_to_token.size(); + + const auto * plogits = logits; + + const auto last_n_tokens = std::vector(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size); + + if (temp <= 0) { + // select the token with the highest logit directly + float max_logit = plogits[0]; + gpt_vocab::id max_id = 0; + + for (int i = 1; i < n_logits; ++i) { + if (plogits[i] > max_logit) { + max_logit = plogits[i]; + max_id = i; + } + } + return max_id; + } + + + std::vector> logits_id; + logits_id.reserve(n_logits); + + { + const float scale = 1.0f/temp; + for (int i = 0; i < n_logits; ++i) { + // repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858) + // credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main + if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) { + // if score < 0 then repetition penalty has to multiplied to reduce the previous token probability + if (plogits[i] < 0.0f) { + logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i)); + } else { + logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i)); + } + } else { + logits_id.push_back(std::make_pair(plogits[i]*scale, i)); + } + } + } + + // find the top K tokens + std::partial_sort( + logits_id.begin(), + logits_id.begin() + top_k, logits_id.end(), + [](const std::pair & a, const std::pair & b) { + return a.first > b.first; + }); + + logits_id.resize(top_k); + + double maxl = -INFINITY; + for (const auto & kv : logits_id) { + maxl = std::max(maxl, kv.first); + } + + // compute probs for the top K tokens + std::vector probs; + probs.reserve(logits_id.size()); + + double sum = 0.0; + for (const auto & kv : logits_id) { + double p = exp(kv.first - maxl); + probs.push_back(p); + sum += p; + } + + // normalize the probs + for (auto & p : probs) { + p /= sum; + } + + if (top_p < 1.0f) { + double cumsum = 0.0f; + for (int i = 0; i < top_k; i++) { + cumsum += probs[i]; + if (cumsum >= top_p) { + top_k = i + 1; + probs.resize(top_k); + logits_id.resize(top_k); + break; + } + } + + cumsum = 1.0/cumsum; + for (int i = 0; i < (int) probs.size(); i++) { + probs[i] *= cumsum; + } + } + +// printf("\n"); +// for (int i = 0; i < (int) probs.size(); i++) { +// for (int i = 0; i < 10; i++) { +// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]); +// } + + std::discrete_distribution<> dist(probs.begin(), probs.end()); + int idx = dist(rng); + + return logits_id[idx].second; + +} diff --git a/gptneox-common.h b/gptneox-common.h new file mode 100644 index 0000000000000..60e5650c129ab --- /dev/null +++ b/gptneox-common.h @@ -0,0 +1,125 @@ +// Various helper functions and utilities + +#pragma once + +#include +#include +#include +#include +#include + +// +// CLI argument parsing +// + +struct gpt_params { + int32_t seed = -1; // RNG seed + int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); + int32_t n_predict = 200; // new tokens to predict + int32_t n_batch = 8; // batch size for prompt processing + + // sampling parameters + int32_t top_k = 40; + float top_p = 0.9f; + float temp = 0.9f; + int32_t repeat_last_n = 64; + float repeat_penalty = 1.00f; + + std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path + std::string prompt = ""; + std::string token_test = ""; + + bool interactive = false; + int32_t interactive_port = -1; + + int32_t n_gpu_layers = 0; +}; + +bool gpt_params_parse(int argc, char ** argv, gpt_params & params); + +void gpt_print_usage(int argc, char ** argv, const gpt_params & params); + +std::string gpt_random_prompt(std::mt19937 & rng); + +// +// Vocab utils +// + +std::string trim(const std::string & s); + +std::string replace( + const std::string & s, + const std::string & from, + const std::string & to); + +struct gpt_vocab { + using id = int32_t; + using token = std::string; + + std::map token_to_id; + std::map id_to_token; + std::vector special_tokens; + + void add_special_token(const std::string & token); +}; + +// poor-man's JSON parsing +std::map json_parse(const std::string & fname); + +std::string convert_to_utf8(const std::wstring & input); + +std::wstring convert_to_wstring(const std::string & input); + +void gpt_split_words(std::string str, std::vector& words); + +// split text into tokens +// +// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53 +// +// Regex (Python): +// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" +// +// Regex (C++): +// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)" +// +std::vector gpt_tokenize(const gpt_vocab & vocab, const std::string & text); + +// test outputs of gpt_tokenize +// +// - compare with tokens generated by the huggingface tokenizer +// - test cases are chosen based on the model's main language (under 'prompt' directory) +// - if all sentences are tokenized identically, print 'All tests passed.' +// - otherwise, print sentence, huggingface tokens, ggml tokens +// +void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test); + +// load the tokens from encoder.json +bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab); + +// sample next token given probabilities for each embedding +// +// - consider only the top K tokens +// - from them, consider only the top tokens with cumulative probability > P +// +// TODO: not sure if this implementation is correct +// TODO: temperature is not implemented +// +gpt_vocab::id gpt_sample_top_k_top_p( + const gpt_vocab & vocab, + const float * logits, + int top_k, + double top_p, + double temp, + std::mt19937 & rng); + +gpt_vocab::id gpt_sample_top_k_top_p_repeat( + const gpt_vocab & vocab, + const float * logits, + const int32_t * last_n_tokens_data, + size_t last_n_tokens_data_size, + int top_k, + double top_p, + double temp, + int repeat_last_n, + float repeat_penalty, + std::mt19937 & rng); From f175b05872462c5a3f14f072945884acef7861f8 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 15:08:37 +0200 Subject: [PATCH 048/242] Makefile : add gptneox gguf example --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e19acfbb24a3f..7d37b66a698ec 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Define the default target now so that it is always the first target -BUILD_TARGETS = main quantize quantize-stats perplexity embedding vdot train-text-from-scratch simple server embd-input-test gguf +BUILD_TARGETS = main quantize quantize-stats perplexity embedding vdot train-text-from-scratch simple server embd-input-test gguf gptneox-main # Binaries only useful for tests TEST_TARGETS = tests/test-double-float tests/test-grad0 tests/test-opt tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0 @@ -329,6 +329,9 @@ grammar-parser.o: examples/grammar-parser.cpp examples/grammar-parser.h libllama.so: llama.o ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -shared -fPIC -o $@ $^ $(LDFLAGS) +gptneox-common.o: gptneox-common.cpp gptneox-common.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + clean: rm -vf *.o *.so *.dll main quantize quantize-stats perplexity embedding benchmark-matmult save-load-state server simple vdot train-text-from-scratch embd-input-test gguf build-info.h $(TEST_TARGETS) @@ -373,6 +376,9 @@ embd-input-test: $(LIB_PRE)embdinput$(DSO_EXT) examples/embd-input/embd-input-te gguf: examples/gguf/gguf.cpp build-info.h ggml.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) +gptneox-main: gptneox-main.cpp gptneox-common.o ggml.o $(OBJS) + $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp build-info.h ggml.o llama.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) From 2fabc176ced5d5cd13458a7856fe67e9189e5b64 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 16:28:08 +0200 Subject: [PATCH 049/242] Update convert-llama-h5-to-gguf.py --- convert-llama-h5-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 926dd265b4a4c..7c867619bc659 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -65,7 +65,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: hparams = json.load(f) # This mmust be changed when adding/deleting kv -kv_count = 13 +kv_count = 14 print("tensors " + str(tensor_count) + " kv " + str(kv_count)) From 30c4ea47e6930c5bf9877d7fd7c468557e960fb4 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 16:59:26 +0200 Subject: [PATCH 050/242] add gptneox gguf example --- gptneox-main.cpp | 812 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 812 insertions(+) create mode 100644 gptneox-main.cpp diff --git a/gptneox-main.cpp b/gptneox-main.cpp new file mode 100644 index 0000000000000..02fbc9fba35a5 --- /dev/null +++ b/gptneox-main.cpp @@ -0,0 +1,812 @@ +#include "ggml.h" + +#include "gptneox-common.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +// default hparams +struct gpt_neox_hparams { + size_t n_merges = 0; + size_t n_vocab = 0; + int32_t n_ctx = 0; + int32_t n_embd = 0; + int32_t n_head = 0; + int32_t n_layer = 0; + int32_t n_rot = 0; // rotary_pct * (n_embd / n_head) + bool par_res = true; + float norm_eps = 1e-5; +}; + +struct gpt_neox_layer { + // pre normalization + struct ggml_tensor * ln_1_g; + struct ggml_tensor * ln_1_b; + + // attention + struct ggml_tensor * c_attn_attn_w; + struct ggml_tensor * c_attn_attn_b; + + struct ggml_tensor * c_attn_proj_w; + struct ggml_tensor * c_attn_proj_b; + + // post normalization + struct ggml_tensor * ln_2_g; + struct ggml_tensor * ln_2_b; + + // ff + struct ggml_tensor * c_mlp_fc_w; + struct ggml_tensor * c_mlp_fc_b; + + struct ggml_tensor * c_mlp_proj_w; + struct ggml_tensor * c_mlp_proj_b; +}; + +struct gpt_neox_model { + gpt_neox_hparams hparams; + + // normalization + struct ggml_tensor * ln_f_g; + struct ggml_tensor * ln_f_b; + + struct ggml_tensor * wte; // position embedding + + struct ggml_tensor * lmh_g; // language model head + + std::vector layers; + + // key + value memory + struct ggml_tensor * memory_k; + struct ggml_tensor * memory_v; + + // + struct gguf_context * ggufctx; + struct ggml_context * ctx; + struct ggml_context * kvctx; + + std::map tensors; +}; + +struct ggml_tensor * get_tensor_ex( struct ggml_context * ctx, std::string name){ + + struct ggml_tensor * cur = ggml_get_tensor(ctx, name.c_str()); + if( cur == NULL ) { + fprintf(stdout, "%s: tensor '%s' not found!\n", __func__, name.c_str()); + } else { +// fprintf(stdout, "%s: n_dims = %d, name = '%s'\n", __func__, cur->n_dims, cur->name); + } + + return cur; +} + +// load the model's weights from a file +bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_vocab & vocab) { + printf("%s: loading model from '%s'..\n", __func__, fname.c_str()); + + model.ctx = NULL; + + struct gguf_init_params ggufparams = { + /*.no_alloc = */ false, + /*.ctx = */ &model.ctx, + }; + + auto & ggufctx = model.ggufctx; + + ggufctx = gguf_init_from_file(fname.c_str(), ggufparams); + + if (!ggufctx) { + fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__); + return false; + } + + fprintf(stdout, "%s: gguf version = %d\n", __func__, gguf_get_version(ggufctx)); + fprintf(stdout, "%s: gguf alignment = %zu\n", __func__, gguf_get_alignment(ggufctx)); + fprintf(stdout, "%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx)); + + // print all kv + if( false ) + { + const int n_kv = gguf_get_n_kv(ggufctx); + + fprintf(stdout, "%s: n_kv: %d\n", __func__, n_kv); + + for (int i = 0; i < n_kv; ++i) { + const char * key = gguf_get_key(ggufctx, i); + + fprintf(stdout, "%s: kv[%d]: key = %s\n", __func__, i, key); + } + } + + // print some standard metadata + { + int keyidx; + + keyidx = gguf_find_key(ggufctx, "general.name"); + if (keyidx != -1) { fprintf(stdout, "%s: model name = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + keyidx = gguf_find_key(ggufctx, "general.description"); + if (keyidx != -1) { fprintf(stdout, "%s: model description = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + keyidx = gguf_find_key(ggufctx, "general.author"); + if (keyidx != -1) { fprintf(stdout, "%s: model author = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + keyidx = gguf_find_key(ggufctx, "general.license"); + if (keyidx != -1) { fprintf(stdout, "%s: model license = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + keyidx = gguf_find_key(ggufctx, "general.architecture"); + if (keyidx != -1) { fprintf(stdout, "%s: model architecture = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + } + + // check required metadata + { + int keyidx; + + keyidx = gguf_find_key(ggufctx, "general.architecture"); + if (keyidx != -1) { + if ( strcmp(gguf_get_val_str(ggufctx, keyidx), "gptneox") != 0) { + fprintf(stdout, "%s: model architecture not supported!\n", __func__); + return false; + } + } else { + fprintf(stdout, "%s: gguf model architecture not found!\n", __func__); + return false; + } + + } + + // load hparams + { + auto & hparams = model.hparams; + + bool ok = true; + int keyidx; + + if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.context_length"); + if (keyidx != -1) { hparams.n_ctx = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } + + if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.embedding_length"); + if (keyidx != -1) { hparams.n_embd = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } + + if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.attention.head_count"); + if (keyidx != -1) { hparams.n_head = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } + + if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.layer_count"); + if (keyidx != -1) { hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } + + if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.rope.dimension_count"); + if (keyidx != -1) { hparams.n_rot = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } + + if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.use_parallel_residual"); + if (keyidx != -1) { hparams.par_res = gguf_get_val_bool(ggufctx, keyidx); } else { ok = false; } } + + if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.attention.layer_norm_epsilon"); + if (keyidx != -1) { hparams.norm_eps= gguf_get_val_f32(ggufctx, keyidx); } else { ok = false; } } + + if (!ok) { + fprintf(stderr, "%s: required hparam missing!\n", __func__); + return false; + } + + printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx); + printf("%s: n_embd = %d\n", __func__, hparams.n_embd); + printf("%s: n_head = %d\n", __func__, hparams.n_head); + printf("%s: n_layer = %d\n", __func__, hparams.n_layer); + printf("%s: n_rot = %d\n", __func__, hparams.n_rot); + printf("%s: par_res = %d\n", __func__, hparams.par_res); + printf("%s: norm_eps = %g\n", __func__, hparams.norm_eps); + + } + + // load vocab + { + + // TODO: implement a better bpe tokenizer, utilizing merges and handles unicode + + auto & hparams = model.hparams; + + int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model"); + + if (keyidx != -1) { + if ( strcmp(gguf_get_val_str(ggufctx, keyidx), "gpt2") != 0) { + fprintf(stdout, "%s: tokenizer model not supported!\n", __func__); + return false; + } + } else { + fprintf(stdout, "%s: tokenizer model not found!\n", __func__); + return false; + } + + + int tokens_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.tokens"); + + if (tokens_keyidx == -1) { + fprintf(stdout, "%s: gpt2 tokenizer vocab not found!\n", __func__); + return false; + } + + int merges_keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.merges"); + + if (merges_keyidx == -1) { + fprintf(stdout, "%s: gpt2 tokenizer merges not found!\n", __func__); + return false; + } + + hparams.n_vocab = gguf_get_arr_n(ggufctx,tokens_keyidx); + hparams.n_merges = gguf_get_arr_n(ggufctx,merges_keyidx); + + fprintf(stdout, "%s: gpt2 tokenizer vocab = %zu\n", __func__, hparams.n_vocab); + fprintf(stdout, "%s: gpt2 tokenizer merges = %zu\n", __func__, hparams.n_merges); + + for (size_t i = 0; i < hparams.n_vocab; i++) { + std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i); + + + // TEMP until a better bpe tokenizer is implemented + word = replace(word, "Ġ", " "); + word = replace(word, "Ċ", "\n"); + + + vocab.token_to_id[word] = i; + vocab.id_to_token[i] = word; + } + + + } + + + auto & ctx = model.ctx; + size_t ctx_size = ggml_get_mem_size(ctx); + + printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0)); + + // print tensor info + if( false ) + { + const int n_tensors = gguf_get_n_tensors(ggufctx); + + fprintf(stdout, "%s: n_tensors: %d\n", __func__, n_tensors); + + for (int i = 0; i < n_tensors; ++i) { + const char * name = gguf_get_tensor_name (ggufctx, i); + const size_t offset = gguf_get_tensor_offset(ggufctx, i); + + fprintf(stdout, "%s: tensor[%d]: name = %s, offset = %zu\n", __func__, i, name, offset); + } + } + + + // prepare memory for the weights + { + const int n_layer = model.hparams.n_layer; + + model.layers.resize(n_layer); + + model.wte = ggml_get_tensor(ctx, "gpt_neox.embed_in.weight"); + model.ln_f_g = ggml_get_tensor(ctx, "gpt_neox.final_layer_norm.weight"); + model.ln_f_b = ggml_get_tensor(ctx, "gpt_neox.final_layer_norm.bias"); + model.lmh_g = ggml_get_tensor(ctx, "embed_out.weight"); + + // map by name + model.tensors["gpt_neox.embed_in.weight"] = model.wte; + model.tensors["gpt_neox.final_layer_norm.weight"] = model.ln_f_g; + model.tensors["gpt_neox.final_layer_norm.bias"] = model.ln_f_b; + model.tensors["embed_out.weight"] = model.lmh_g; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = model.layers[i]; + + layer.ln_1_g = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".input_layernorm.weight" ); + layer.ln_1_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".input_layernorm.bias" ); + + layer.c_attn_attn_w = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".attention.query_key_value.weight" ); + layer.c_attn_attn_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".attention.query_key_value.bias" ); + + layer.c_attn_proj_w = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".attention.dense.weight" ); + layer.c_attn_proj_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".attention.dense.bias" ); + + layer.ln_2_g = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".post_attention_layernorm.weight" ); + layer.ln_2_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".post_attention_layernorm.bias"); + + layer.c_mlp_fc_w = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".mlp.dense_h_to_4h.weight" ); + layer.c_mlp_fc_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".mlp.dense_h_to_4h.bias" ); + + layer.c_mlp_proj_w = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".mlp.dense_4h_to_h.weight" ); + layer.c_mlp_proj_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".mlp.dense_4h_to_h.bias" ); + + // map by name + model.tensors["gpt_neox.layers." + std::to_string(i) + ".input_layernorm.weight"] = layer.ln_1_g; + model.tensors["gpt_neox.layers." + std::to_string(i) + ".input_layernorm.bias"] = layer.ln_1_b; + + model.tensors["gpt_neox.layers." + std::to_string(i) + ".attention.query_key_value.weight"] = layer.c_attn_attn_w; + model.tensors["gpt_neox.layers." + std::to_string(i) + ".attention.query_key_value.bias"] = layer.c_attn_attn_b; + + model.tensors["gpt_neox.layers." + std::to_string(i) + ".attention.dense.weight"] = layer.c_attn_proj_w; + model.tensors["gpt_neox.layers." + std::to_string(i) + ".attention.dense.bias"] = layer.c_attn_proj_b; + + model.tensors["gpt_neox.layers." + std::to_string(i) + ".post_attention_layernorm.weight"] = layer.ln_2_g; + model.tensors["gpt_neox.layers." + std::to_string(i) + ".post_attention_layernorm.bias"] = layer.ln_2_b; + + model.tensors["gpt_neox.layers." + std::to_string(i) + ".mlp.dense_h_to_4h.weight"] = layer.c_mlp_fc_w; + model.tensors["gpt_neox.layers." + std::to_string(i) + ".mlp.dense_h_to_4h.bias"] = layer.c_mlp_fc_b; + + model.tensors["gpt_neox.layers." + std::to_string(i) + ".mlp.dense_4h_to_h.weight"] = layer.c_mlp_proj_w; + model.tensors["gpt_neox.layers." + std::to_string(i) + ".mlp.dense_4h_to_h.bias"] = layer.c_mlp_proj_b; + } + } + + // key + value memory + { + const auto & kvctx = model.kvctx; + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + + const int64_t n_mem = n_layer*n_ctx; + const int64_t n_elements = n_embd*n_mem; + + // create the ggml context + { + struct ggml_init_params params = { + /*.mem_size =*/ size_t(n_elements*4+ggml_tensor_overhead()*2), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + + model.kvctx = ggml_init(params); + if (!model.kvctx) { + fprintf(stderr, "%s: kv ggml_init() failed\n", __func__); + return false; + } + + } + + + model.memory_k = ggml_new_tensor_1d(kvctx, GGML_TYPE_F16, n_elements); + model.memory_v = ggml_new_tensor_1d(kvctx, GGML_TYPE_F16, n_elements); + + const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v); + + printf("%s: memory_size = %8.2f MB, n_mem = %" PRId64 "\n", __func__, memory_size/1024.0/1024.0, n_mem); + } + + return true; +} + + +// feed-forward network +ggml_tensor * gpt_neox_ff( + const gpt_neox_layer &layer, + ggml_context * ctx0, + ggml_tensor * inp) { + ggml_tensor * cur = ggml_norm(ctx0, inp); + + cur = ggml_add(ctx0, + ggml_mul(ctx0, + ggml_repeat(ctx0, layer.ln_2_g, cur), + cur), + ggml_repeat(ctx0, layer.ln_2_b, cur)); + + cur = ggml_mul_mat(ctx0, + layer.c_mlp_fc_w, + cur); + + cur = ggml_add(ctx0, + ggml_repeat(ctx0, layer.c_mlp_fc_b, cur), + cur); + + // GELU activation + cur = ggml_gelu(ctx0, cur); + + // projection + // cur = proj_w*cur + proj_b + cur = ggml_mul_mat(ctx0, + layer.c_mlp_proj_w, + cur); + + cur = ggml_add(ctx0, + ggml_repeat(ctx0, layer.c_mlp_proj_b, cur), + cur); + return cur; +} + +// evaluate the transformer +// +// - model: the model +// - n_threads: number of threads to use +// - n_past: the context size so far +// - embd_inp: the embeddings of the tokens in the context +// - embd_w: the predicted logits for the next token +// +bool gpt_neox_eval( + const gpt_neox_model & model, + const int n_threads, + const int n_past, + const std::vector & embd_inp, + std::vector & embd_w, + size_t & mem_per_token) { + const int N = embd_inp.size(); + + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_head = hparams.n_head; + const int n_vocab = hparams.n_vocab; + const int n_rot = hparams.n_rot; + + static size_t buf_size = 256u*1024*1024; + static void * buf = malloc(buf_size); + + // use 2 scratch buffers + // TODO: very hacky solution - reimplement in a more elegant way + static size_t scr0_size = 256u*1024*1024; + static void * scr0 = malloc(scr0_size); + + static size_t scr1_size = 256u*1024*1024; + static void * scr1 = malloc(scr1_size); + + if (mem_per_token > 0 && mem_per_token*N > buf_size) { + const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead + //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new); + + // reallocate + buf_size = buf_size_new; + buf = realloc(buf, buf_size); + if (buf == nullptr) { + fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size); + return false; + } + } + + struct ggml_init_params params = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ buf, + /*.no_alloc =*/ false, + }; + + struct ggml_context * ctx0 = ggml_init(params); + struct ggml_cgraph gf = {}; + + struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); + memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd)); + + + // wte + struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * cur; + + ggml_set_scratch(ctx0, { 0, scr0_size, scr0, }); + + // self-attention + { + { + cur = ggml_norm(ctx0, inpL); + + cur = ggml_add(ctx0, + ggml_mul(ctx0, + ggml_repeat(ctx0, model.layers[il].ln_1_g, cur), + cur), + ggml_repeat(ctx0, model.layers[il].ln_1_b, cur)); + } + + // compute QKV + { + + cur = ggml_mul_mat(ctx0, + model.layers[il].c_attn_attn_w, + cur); + + cur = ggml_add(ctx0, + ggml_repeat(ctx0, model.layers[il].c_attn_attn_b, cur), + cur); + } + + struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd/n_head, n_head, N, cur->nb[1]/n_head, cur->nb[1], 0*sizeof(float)*n_embd/n_head)); + struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd/n_head, n_head, N, cur->nb[1]/n_head, cur->nb[1], 1*sizeof(float)*n_embd/n_head)); + struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd/n_head, n_head, N, cur->nb[1]/n_head, cur->nb[1], 2*sizeof(float)*n_embd/n_head)); + + // using mode = 2 for GPT-NeoX mode + Qcur = ggml_rope_inplace(ctx0, Qcur, n_past, n_rot, 2, 0); + Kcur = ggml_rope_inplace(ctx0, Kcur, n_past, n_rot, 2, 0); + + // store key and value to memory + { + Vcur = ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd, N)); + + struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past)); + struct ggml_tensor * v = ggml_view_2d(ctx0, model.memory_v, N, n_embd, + ( n_ctx)*ggml_element_size(model.memory_v), + (il*n_ctx)*ggml_element_size(model.memory_v)*n_embd + n_past*ggml_element_size(model.memory_v)); + + ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Kcur, k)); + ggml_build_forward_expand(&gf, ggml_cpy(ctx0, Vcur, v)); + } + + // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3) + struct ggml_tensor * Q = + ggml_permute(ctx0, + Qcur, + 0, 2, 1, 3); + + // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3) + struct ggml_tensor * K = + ggml_permute(ctx0, + ggml_reshape_3d(ctx0, + ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd), + n_embd/n_head, n_head, n_past + N), + 0, 2, 1, 3); + + // K * Q + struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); + + // KQ_scaled = KQ / sqrt(n_embd/n_head) + struct ggml_tensor * KQ_scaled = + ggml_scale_inplace(ctx0, + KQ, + ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head)) + ); + + // KQ_masked = mask_past(KQ_scaled) + struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past); + + // KQ = soft_max(KQ_masked) + struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked); + + // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous() + struct ggml_tensor * V = + ggml_view_3d(ctx0, model.memory_v, + n_past + N, n_embd/n_head, n_head, + n_ctx*ggml_element_size(model.memory_v), + n_ctx*ggml_element_size(model.memory_v)*n_embd/n_head, + il*n_ctx*ggml_element_size(model.memory_v)*n_embd); + + // KQV = transpose(V) * KQ_soft_max + struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max); + + // KQV_merged = KQV.permute(0, 2, 1, 3) + struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3); + + // cur = KQV_merged.contiguous().view(n_embd, N) + cur = ggml_cpy(ctx0, + KQV_merged, + ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N)); + + // projection + { + cur = ggml_mul_mat(ctx0, + model.layers[il].c_attn_proj_w, + cur); + + cur = ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].c_attn_proj_b, cur), cur); + } + } + + ggml_set_scratch(ctx0, { 0, scr1_size, scr1, }); + + if (hparams.par_res == 0) { + struct ggml_tensor * inpFF = ggml_add(ctx0, cur, inpL); + + cur = gpt_neox_ff(model.layers[il], ctx0, inpFF); + + // input for next layer + inpL = ggml_add(ctx0, cur, inpFF); + } else { + struct ggml_tensor * inpFF = cur; + + // this is independent of the self-attention result, so it could be done in parallel to the self-attention + // note here we pass inpL instead of cur + cur = gpt_neox_ff(model.layers[il], ctx0, inpL); + + // layer input + FF + cur = ggml_add(ctx0, cur, inpFF); + + // input for next layer + inpL = ggml_add(ctx0, cur, inpL); + } + } + + ggml_set_scratch(ctx0, { 0, scr0_size, scr0, }); + + // norm + { + inpL = ggml_norm(ctx0, inpL); + + // inpL = ln_f_g*inpL + ln_f_b + inpL = ggml_add(ctx0, + ggml_mul(ctx0, + ggml_repeat(ctx0, model.ln_f_g, inpL), + inpL), + ggml_repeat(ctx0, model.ln_f_b, inpL)); + } + + ggml_set_scratch(ctx0, { 0, 0, nullptr, }); + + // lm_head + { + inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL); + + //inpL = ggml_add(ctx0, + // ggml_repeat(ctx0, model.lmh_b, inpL), + // inpL); + } + + // logits -> probs + //inpL = ggml_soft_max_inplace(ctx0, inpL); + + // run the computation + ggml_build_forward_expand(&gf, inpL); + ggml_graph_compute_with_ctx(ctx0, &gf, n_threads); + + //if (n_past%100 == 0) { + // ggml_graph_print (&gf); + // ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot"); + //} + + //embd_w.resize(n_vocab*N); + //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N); + + // return result for just the last token + embd_w.resize(n_vocab); + memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab); + + if (mem_per_token == 0) { + mem_per_token = ggml_used_mem(ctx0)/N; + } + //printf("used_mem = %zu\n", ggml_used_mem(ctx0)); + + ggml_free(ctx0); + + return true; +} + +int main(int argc, char ** argv) { + ggml_time_init(); + + const int64_t t_main_start_us = ggml_time_us(); + + gpt_params params; + + if (gpt_params_parse(argc, argv, params) == false) { + return 1; + } + + if (params.seed < 0) { + params.seed = time(NULL); + } + + printf("%s: seed = %d\n", __func__, params.seed); + + std::mt19937 rng(params.seed); + if (params.prompt.empty()) { + params.prompt = gpt_random_prompt(rng); + } + + int64_t t_load_us = 0; + + gpt_vocab vocab; + gpt_neox_model model; + + // load the model + { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt_neox_model_load(params.model, model, vocab)) { + fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); + return 1; + } + + t_load_us = ggml_time_us() - t_start_us; + + } + + int n_past = 0; + + int64_t t_sample_us = 0; + int64_t t_predict_us = 0; + + std::vector logits; + + // tokenize the prompt + std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt); + + params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size()); + + printf("%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size()); + for (int i = 0; i < embd_inp.size(); i++) { + printf("%s: token[%d] = %6d, %s\n", __func__, i, embd_inp[i], vocab.id_to_token.at(embd_inp[i]).c_str()); + } + printf("\n"); + + std::vector embd; + + // determine the required inference memory per token: + size_t mem_per_token = 0; + gpt_neox_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token); + + for (int i = embd.size(); i < embd_inp.size() + params.n_predict; i++) { + // predict + if (embd.size() > 0) { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt_neox_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) { + printf("Failed to predict\n"); + return 1; + } + + t_predict_us += ggml_time_us() - t_start_us; + } + + n_past += embd.size(); + embd.clear(); + + if (i >= embd_inp.size()) { + // sample next token + const int top_k = params.top_k; + const float top_p = params.top_p; + const float temp = params.temp; + + const int n_vocab = model.hparams.n_vocab; + + gpt_vocab::id id = 0; + + { + const int64_t t_start_sample_us = ggml_time_us(); + + id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng); + + t_sample_us += ggml_time_us() - t_start_sample_us; + } + + // add it to the context + embd.push_back(id); + } else { + // if here, it means we are still processing the input prompt + for (int k = i; k < embd_inp.size(); k++) { + embd.push_back(embd_inp[k]); + if (embd.size() > params.n_batch) { + break; + } + } + i += embd.size() - 1; + } + + // display text + for (auto id : embd) { + printf("%s", vocab.id_to_token[id].c_str()); + } + fflush(stdout); + + // end of text token + if (embd.back() == 0) { + break; + } + } + + // report timing + { + const int64_t t_main_end_us = ggml_time_us(); + + printf("\n\n"); + printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token); + printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f); + printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f); + printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past); + printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); + } + + ggml_free(model.ctx); + + return 0; +} From 068a8e0fbe126b7cbd1e4cf2fc125c8246513315 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 17:29:56 +0200 Subject: [PATCH 051/242] Update convert-llama-h5-to-gguf.py --- convert-llama-h5-to-gguf.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 7c867619bc659..3e7529b2faa89 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -64,8 +64,8 @@ def permute(weights: NDArray, n_head: int) -> NDArray: with open(dir_model + "/config.json", "r", encoding="utf-8") as f: hparams = json.load(f) -# This mmust be changed when adding/deleting kv -kv_count = 14 +# This must be changed when adding/deleting kv +kv_count = 13 print("tensors " + str(tensor_count) + " kv " + str(kv_count)) @@ -86,7 +86,6 @@ def permute(weights: NDArray, n_head: int) -> NDArray: gguf_writer.write_feed_forward_length(llm_arch, hparams["intermediate_size"]) gguf_writer.write_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"]) gguf_writer.write_head_count(llm_arch, hparams["num_attention_heads"]) -gguf_writer.write_float32(llm_arch + ".attention.layer_norm_rms_epsilon", hparams["rms_norm_eps"]) gguf_writer.write_layer_norm_rms_eps(llm_arch, hparams["rms_norm_eps"]) From 2a0914673cdd57e0388b69711f04f7b9029d7bfe Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 30 Jul 2023 17:31:11 +0200 Subject: [PATCH 052/242] Update convert-gptneox-h5-to-gguf.py --- convert-gptneox-h5-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index cbade7079be8b..12b6cc960775a 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -54,7 +54,7 @@ with open(dir_model + "/config.json", "r", encoding="utf-8") as f: hparams = json.load(f) -# This mmust be changed when adding/deleting kv +# This must be changed when adding/deleting kv kv_count = 14 print("tensors " + str(tensor_count) + " kv " + str(kv_count)) From 4f5b6224beb7b1a0291843cb118b148d5b6e7a6c Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 31 Jul 2023 03:00:20 +0200 Subject: [PATCH 053/242] Update convert-gptneox-h5-to-gguf.py --- convert-gptneox-h5-to-gguf.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index 12b6cc960775a..2661995096979 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -37,6 +37,12 @@ sys.exit(1) fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" +with open(dir_model + "/config.json", "r", encoding="utf-8") as f: + hparams = json.load(f) + +if hparams["architectures"][0] != "GPTNeoXForCausalLM": + print("Model architecture not supported: " + hparams["architectures"][0] ) + sys.exit() model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) list_vars = model.state_dict() @@ -51,9 +57,6 @@ gguf_writer = gguf.GGUFWriter.open(fname_out) -with open(dir_model + "/config.json", "r", encoding="utf-8") as f: - hparams = json.load(f) - # This must be changed when adding/deleting kv kv_count = 14 From 6b3a7b9f4f726a7a3474e1538d74d3e8d6a92541 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 31 Jul 2023 03:02:00 +0200 Subject: [PATCH 054/242] Update convert-llama-h5-to-gguf.py --- convert-llama-h5-to-gguf.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 3e7529b2faa89..0451ffe232b85 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -47,6 +47,12 @@ def permute(weights: NDArray, n_head: int) -> NDArray: sys.exit(1) fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" +if hparams["architectures"][0] != "LlamaForCausalLM": + print("Model architecture not supported: " + hparams["architectures"][0] ) + sys.exit() + +with open(dir_model + "/config.json", "r", encoding="utf-8") as f: + hparams = json.load(f) model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) list_vars = model.state_dict() @@ -61,9 +67,6 @@ def permute(weights: NDArray, n_head: int) -> NDArray: gguf_writer = gguf.GGUFWriter.open(fname_out) -with open(dir_model + "/config.json", "r", encoding="utf-8") as f: - hparams = json.load(f) - # This must be changed when adding/deleting kv kv_count = 13 From 7aa0a0e7f7ff7d7809a289475f77999dcfe093b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Mon, 31 Jul 2023 09:59:36 +0300 Subject: [PATCH 055/242] gguf : support custom alignment value --- constants.py | 1 + ggml.c | 5 ++++- gguf.py | 6 +++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/constants.py b/constants.py index 14f11123b1280..d9e110b73fd63 100644 --- a/constants.py +++ b/constants.py @@ -5,6 +5,7 @@ # general KEY_GENERAL_ARCHITECTURE = "general.architecture" KEY_GENERAL_QUANTIZATION_VERSION = "general.quantization_version" +KEY_GENERAL_ALIGNMENT = "general.alignment" KEY_GENERAL_NAME = "general.name" KEY_GENERAL_AUTHOR = "general.author" KEY_GENERAL_URL = "general.url" diff --git a/ggml.c b/ggml.c index 1b2f071e71a3d..157199118b282 100644 --- a/ggml.c +++ b/ggml.c @@ -18543,7 +18543,10 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p ctx->alignment = GGUF_DEFAULT_ALIGNMENT; - // TODO: determine new alignment from kv if available + int alignment_idx = gguf_find_key(ctx, "general.alignment"); + if (alignment_idx != -1) { + ctx->alignment = gguf_get_u32(ctx, alignment_idx); + } // we require the data section to be aligned, so take into account any padding { diff --git a/gguf.py b/gguf.py index 99ea817028c70..8e2c771baebc5 100644 --- a/gguf.py +++ b/gguf.py @@ -220,6 +220,9 @@ def write_quantization_version(self, quantization_version: GGMLQuantizationType) self.write_uint32( constants.KEY_GENERAL_QUANTIZATION_VERSION, quantization_version) + def write_custom_alignment(self, alignment: int): + self.write_uint32(constants.KEY_GENERAL_ALIGNMENT, alignment) + def write_context_length(self, llm: str, length: int): self.write_uint32( constants.KEY_LLM_CONTEXT_LENGTH.format(llm=llm), length) @@ -292,11 +295,12 @@ def write_token_scores(self, scores: List[float]): if __name__ == "__main__": # Example usage with a file gguf_writer = GGUFWriter.open("example.gguf") - gguf_writer.write_header(2, 3) + gguf_writer.write_header(2, 4) gguf_writer.write_architecture("llama") gguf_writer.write_uint32("answer", 42) # Write a 32-bit integer gguf_writer.write_float32("answer_in_float", 42.0) # Write a 32-bit float + gguf_writer.write_custom_alignment(64) tensor1 = np.ones((32,), dtype=np.float32) * 100.0 tensor2 = np.ones((32,), dtype=np.float32) * 101.0 gguf_writer.write_tensor_info("tensor0", tensor1) From b26f5b2e43eb3e8b1f5948f16eafe22b362e6c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Mon, 31 Jul 2023 16:23:54 +0300 Subject: [PATCH 056/242] gguf : fix typo in function call --- ggml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index 157199118b282..33a90f5371f7d 100644 --- a/ggml.c +++ b/ggml.c @@ -18545,7 +18545,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p int alignment_idx = gguf_find_key(ctx, "general.alignment"); if (alignment_idx != -1) { - ctx->alignment = gguf_get_u32(ctx, alignment_idx); + ctx->alignment = gguf_get_val_u32(ctx, alignment_idx); } // we require the data section to be aligned, so take into account any padding From bb42aefaeb115a882d839a3c658ab119b692e075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Mon, 31 Jul 2023 17:46:12 +0300 Subject: [PATCH 057/242] gguf : mmap tensor data example --- examples/gguf/gguf.cpp | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index 84388c219f5ef..deb4a83592779 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -1,4 +1,5 @@ #include "ggml.h" +#include "llama-util.h" #include #include @@ -376,7 +377,31 @@ bool gguf_ex_read_2(const std::string & fname) { // TODO: mmap based on tensor infos - fprintf(stdout, "%s: ctx_data size: %zu\n", __func__, ggml_get_mem_size(ctx_data)); + + struct llama_file file(fname.c_str(), "rb"); + llama_mmap data_mmap(&file, 0, false); + const int n_tensors = gguf_get_n_tensors(ctx); + + for (int i = 0; i < n_tensors; ++i) { + const char * name = gguf_get_tensor_name(ctx, i); + const size_t offset = gguf_get_data_offset(ctx) + gguf_get_tensor_offset(ctx, i); + struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); + + cur->data = static_cast(data_mmap.addr) + offset; + + // print first 10 elements + const float * data = (const float *) cur->data; + + printf("%s data[:10] : ", name); + + for (int j = 0; j < 10; ++j) { + printf("%f ", data[j]); + } + + printf("\n\n"); + } + +fprintf(stdout, "%s: ctx_data size: %zu\n", __func__, ggml_get_mem_size(ctx_data)); ggml_free(ctx_data); gguf_free(ctx); From f3de876a1212767c9ba9223de26472b40f9043ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Mon, 31 Jul 2023 23:58:29 +0300 Subject: [PATCH 058/242] fix : update convert-llama-h5-to-gguf.py --- convert-llama-h5-to-gguf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 0451ffe232b85..412d334fb8920 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -46,13 +46,13 @@ def permute(weights: NDArray, n_head: int) -> NDArray: print("Invalid ftype: " + str(ftype)) sys.exit(1) fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" + +with open(dir_model + "/config.json", "r", encoding="utf-8") as f: + hparams = json.load(f) if hparams["architectures"][0] != "LlamaForCausalLM": print("Model architecture not supported: " + hparams["architectures"][0] ) sys.exit() - -with open(dir_model + "/config.json", "r", encoding="utf-8") as f: - hparams = json.load(f) model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) list_vars = model.state_dict() From da4900e8359aba5923ac4f005608b292b77c45da Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 31 Jul 2023 23:04:03 +0200 Subject: [PATCH 059/242] Update convert-llama-h5-to-gguf.py --- convert-llama-h5-to-gguf.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 412d334fb8920..7f16905959b73 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -104,11 +104,6 @@ def permute(weights: NDArray, n_head: int) -> NDArray: print("Adding sentencepiece tokenizer vocab.") tokenizer = SentencePieceProcessor(dir_model + "/tokenizer.model") - # output vocab_size followed by all piece/score pairs - outbytes: bytes - outbytes = b"" - outbytes += struct.pack("I", tokenizer.vocab_size()) - for i in range(tokenizer.vocab_size()): text: bytes if tokenizer.is_unknown(i): From e7a741695c5a23ef6d591555e7cc255f9b79d690 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:30:00 +0200 Subject: [PATCH 060/242] convert-gptneox-h5-to-gguf.py : Special tokens --- convert-gptneox-h5-to-gguf.py | 42 +++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index 2661995096979..f255cb7d7ee13 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -58,7 +58,7 @@ gguf_writer = gguf.GGUFWriter.open(fname_out) # This must be changed when adding/deleting kv -kv_count = 14 +kv_count = 17 print("tensors " + str(tensor_count) + " kv " + str(kv_count)) @@ -101,9 +101,43 @@ merges = tokenizer["model"]["merges"] -gguf_writer.write_tokenizer_model("gpt2") -gguf_writer.write_token_list(tokens) -gguf_writer.write_token_merges(merges) + gguf_writer.write_tokenizer_model("gpt2") + gguf_writer.write_token_list(tokens) + gguf_writer.write_token_merges(merges) + + if "added_tokens" in tokenizer and Path(dir_model + "/tokenizer_config.json").is_file(): + print("Adding special token ids") + + with open(dir_model + "/tokenizer_config.json", "r", encoding="utf-8") as f: + tokenizer_config = json.load(f) + + # find special token ids + + if "bos_token" in tokenizer_config: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["bos_token"]: + gguf_writer.write_uint32("tokenizer.ggml.bos_token_id", key["id"] ) + + if "eos_token" in tokenizer_config: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["eos_token"]: + gguf_writer.write_uint32("tokenizer.ggml.eos_token_id", key["id"] ) + + if "unk_token" in tokenizer_config: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["unk_token"]: + gguf_writer.write_uint32("tokenizer.ggml.unknown_token_id", key["id"] ) + + if "sep_token" in tokenizer_config: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["sep_token"]: + gguf_writer.write_uint32("tokenizer.ggml.separator_token_id", key["id"] ) + + if "pad_token" in tokenizer_config: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["pad_token"]: + gguf_writer.write_uint32("tokenizer.ggml.padding_token_id", key["id"] ) + # TENSORS From c77fabb1f9b77d6665065da494cb918af71c1dc0 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:32:53 +0200 Subject: [PATCH 061/242] gptneox-main.cpp : special tokens --- gptneox-main.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gptneox-main.cpp b/gptneox-main.cpp index 02fbc9fba35a5..fa95cb7722a11 100644 --- a/gptneox-main.cpp +++ b/gptneox-main.cpp @@ -257,6 +257,11 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_ vocab.id_to_token[i] = word; } + keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.bos_token_id"); if( keyidx != -1 ) { printf("bos id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); } + keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.eos_token_id"); if( keyidx != -1 ) { printf("eos id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); } + keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.unknown_token_id"); if( keyidx != -1 ) { printf("unk id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); } + keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.separator_token_id"); if( keyidx != -1 ) { printf("sep id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); } + keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.padding_token_id"); if( keyidx != -1 ) { printf("pad id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); } } From 36a36c32a3814c036b6290735ba9d4c9111653c7 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:44:28 +0200 Subject: [PATCH 062/242] Update gptneox-main.cpp --- gptneox-main.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gptneox-main.cpp b/gptneox-main.cpp index fa95cb7722a11..235fca7ce7094 100644 --- a/gptneox-main.cpp +++ b/gptneox-main.cpp @@ -716,6 +716,9 @@ int main(int argc, char ** argv) { } + uint32_t eos_token_id = 0; + int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.eos_token_id"); if( keyidx != -1 ) { eos_token_id = gguf_get_val_u32(ggufctx, keyidx); } + int n_past = 0; int64_t t_sample_us = 0; @@ -794,7 +797,7 @@ int main(int argc, char ** argv) { fflush(stdout); // end of text token - if (embd.back() == 0) { + if (embd.back() == eos_token_id) { break; } } From ff1cb02397002f874bde7d91256bb31a6729a577 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Tue, 1 Aug 2023 23:17:21 +0200 Subject: [PATCH 063/242] constants.py : special tokens --- constants.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/constants.py b/constants.py index d9e110b73fd63..ae6e719fb555e 100644 --- a/constants.py +++ b/constants.py @@ -47,3 +47,8 @@ KEY_TOKENIZER_PAD_ID = "tokenizer.ggml.padding_token_id" KEY_TOKENIZER_HF_JSON = "tokenizer.huggingface.json" KEY_TOKENIZER_RWKV = "tokenizer.rwkv.world" +KEY_TOKENIZER_BOS_ID = "tokenizer.ggml.bos_token_id" +KEY_TOKENIZER_EOS_ID = "tokenizer.ggml.eos_token_id" +KEY_TOKENIZER_UNK_ID = "tokenizer.ggml.unknown_token_id" +KEY_TOKENIZER_SEP_ID = "tokenizer.ggml.separator_token_id" +KEY_TOKENIZER_PAD_ID = "tokenizer.ggml.padding_token_id" From 49380a23a32067b392dc1f81857c2129471456a9 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Tue, 1 Aug 2023 23:37:48 +0200 Subject: [PATCH 064/242] gguf.py : accumulate kv and tensor info data + special tokens --- gguf.py | 284 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 159 insertions(+), 125 deletions(-) diff --git a/gguf.py b/gguf.py index 8e2c771baebc5..88e2bee0733b1 100644 --- a/gguf.py +++ b/gguf.py @@ -61,98 +61,113 @@ class GGUFWriter: def __init__(self, fout: IO): self.fout = fout self.offset_tensor = 0 + self.kv_data = b"" + self.kv_data_count = 0 + self.ti_data = b"" + self.ti_data_count = 0 - def write_header(self, tensor_count: int, metadata_kv_count: int): + def write_header_to_file(self): self.fout.write(struct.pack(" "GGUFWriter": f = open(path, "wb") return cls(f) - def write_key(self, key: str): - self.write_val(key, GGUFValueType.STRING, write_vtype=False) + def add_key(self, key: str): + self.add_val(key, GGUFValueType.STRING, add_vtype=False) - def write_uint8(self, key: str, val: int): - self.write_key(key) - self.write_val(val, GGUFValueType.UINT8) + def add_uint8(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.UINT8) - def write_int8(self, key: str, val: int): - self.write_key(key) - self.write_val(val, GGUFValueType.INT8) + def add_int8(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.INT8) - def write_uint16(self, key: str, val: int): - self.write_key(key) - self.write_val(val, GGUFValueType.UINT16) + def add_uint16(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.UINT16) - def write_int16(self, key: str, val: int): - self.write_key(key) - self.write_val(val, GGUFValueType.INT16) + def add_int16(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.INT16) - def write_uint32(self, key: str, val: int): - self.write_key(key) - self.write_val(val, GGUFValueType.UINT32) + def add_uint32(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.UINT32) - def write_int32(self, key: str, val: int): - self.write_key(key) - self.write_val(val, GGUFValueType.INT32) + def add_int32(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.INT32) - def write_float32(self, key: str, val: float): - self.write_key(key) - self.write_val(val, GGUFValueType.FLOAT32) + def add_float32(self, key: str, val: float): + self.add_key(key) + self.add_val(val, GGUFValueType.FLOAT32) - def write_bool(self, key: str, val: bool): - self.write_key(key) - self.write_val(val, GGUFValueType.BOOL) + def add_bool(self, key: str, val: bool): + self.add_key(key) + self.add_val(val, GGUFValueType.BOOL) - def write_string(self, key: str, val: str): - self.write_key(key) - self.write_val(val, GGUFValueType.STRING) + def add_string(self, key: str, val: str): + self.add_key(key) + self.add_val(val, GGUFValueType.STRING) - def write_array(self, key: str, val: list): + def add_array(self, key: str, val: list): if not isinstance(val, list): raise ValueError("Value must be a list for array type") - self.write_key(key) - self.write_val(val, GGUFValueType.ARRAY) + self.add_key(key) + self.add_val(val, GGUFValueType.ARRAY) - def write_val(self: str, val: Any, vtype: GGUFValueType = None, write_vtype: bool = True): + def add_val(self: str, val: Any, vtype: GGUFValueType = None, add_vtype: bool = True): if vtype is None: vtype = GGUFValueType.get_type(val) - if write_vtype: - self.fout.write(struct.pack(" int: return ((x + n - 1) // n) * n - def write_tensor_info(self, name: str, tensor: np.ndarray): - self.write_key(name) + def add_tensor_info(self, name: str, tensor: np.ndarray): + encoded_name = name.encode("utf8") + self.ti_data += struct.pack(" Date: Tue, 1 Aug 2023 23:40:50 +0200 Subject: [PATCH 065/242] convert-gptneox-h5-to-gguf.py : accumulate kv and ti + special tokens --- convert-gptneox-h5-to-gguf.py | 88 ++++++++++++++++------------------- 1 file changed, 39 insertions(+), 49 deletions(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index f255cb7d7ee13..4f9d41487d22f 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -47,51 +47,35 @@ model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) list_vars = model.state_dict() -# count tensors to be converted -tensor_count = 0 -for name in list_vars.keys(): - # we don't need these - if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): - continue - tensor_count += 1 - gguf_writer = gguf.GGUFWriter.open(fname_out) -# This must be changed when adding/deleting kv -kv_count = 17 - -print("tensors " + str(tensor_count) + " kv " + str(kv_count)) - -print("write gguf header") - -gguf_writer.write_header(tensor_count, kv_count) -print("write gguf hparams") +print("gguf: add key-values, metadata") llm_arch = "gptneox" -gguf_writer.write_name("pythia-70b-deduped") -gguf_writer.write_description("gguf test model") -gguf_writer.write_architecture(llm_arch) -gguf_writer.write_context_length(llm_arch, hparams["max_position_embeddings"]) -gguf_writer.write_embedding_length(llm_arch, hparams["hidden_size"]) -gguf_writer.write_layer_count(llm_arch, hparams["num_hidden_layers"]) -gguf_writer.write_feed_forward_length(llm_arch, hparams["intermediate_size"]) -gguf_writer.write_rope_dimension_count(llm_arch, int( hparams["rotary_pct"]*(hparams["hidden_size"]//hparams["num_attention_heads"])) ) -gguf_writer.write_head_count(llm_arch, hparams["num_attention_heads"]) -gguf_writer.write_parallel_residual(llm_arch, hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True) -gguf_writer.write_layer_norm_eps(llm_arch, hparams["layer_norm_eps"]) +gguf_writer.add_name("pythia-70b-deduped") +gguf_writer.add_description("gguf test model") +gguf_writer.add_architecture(llm_arch) +gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) +gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) +gguf_writer.add_layer_count(llm_arch, hparams["num_hidden_layers"]) +gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"]) +gguf_writer.add_rope_dimension_count(llm_arch, int( hparams["rotary_pct"]*(hparams["hidden_size"]//hparams["num_attention_heads"])) ) +gguf_writer.add_head_count(llm_arch, hparams["num_attention_heads"]) +gguf_writer.add_parallel_residual(llm_arch, hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True) +gguf_writer.add_layer_norm_eps(llm_arch, hparams["layer_norm_eps"]) # TOKENIZATION -print("write gguf tokenizer") +print("gguf: add key-values, tokenizer") tokens: List[str] = [] merges: List[str] = [] if Path(dir_model + "/tokenizer.json").is_file(): # vocab type gpt2 - print("Adding gpt2 tokenizer vocab") + print("gguf: adding gpt2 tokenizer vocab") with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f: tokenizer = json.load(f) @@ -101,12 +85,12 @@ merges = tokenizer["model"]["merges"] - gguf_writer.write_tokenizer_model("gpt2") - gguf_writer.write_token_list(tokens) - gguf_writer.write_token_merges(merges) + gguf_writer.add_tokenizer_model("gpt2") + gguf_writer.add_token_list(tokens) + gguf_writer.add_token_merges(merges) if "added_tokens" in tokenizer and Path(dir_model + "/tokenizer_config.json").is_file(): - print("Adding special token ids") + print("gguf: adding special token ids") with open(dir_model + "/tokenizer_config.json", "r", encoding="utf-8") as f: tokenizer_config = json.load(f) @@ -116,33 +100,33 @@ if "bos_token" in tokenizer_config: for key in tokenizer["added_tokens"]: if key["content"] == tokenizer_config["bos_token"]: - gguf_writer.write_uint32("tokenizer.ggml.bos_token_id", key["id"] ) + gguf_writer.add_bos_token_id(key["id"]) if "eos_token" in tokenizer_config: for key in tokenizer["added_tokens"]: if key["content"] == tokenizer_config["eos_token"]: - gguf_writer.write_uint32("tokenizer.ggml.eos_token_id", key["id"] ) + gguf_writer.add_eos_token_id(key["id"]) if "unk_token" in tokenizer_config: for key in tokenizer["added_tokens"]: if key["content"] == tokenizer_config["unk_token"]: - gguf_writer.write_uint32("tokenizer.ggml.unknown_token_id", key["id"] ) + gguf_writer.add_unk_token_id(key["id"]) if "sep_token" in tokenizer_config: for key in tokenizer["added_tokens"]: if key["content"] == tokenizer_config["sep_token"]: - gguf_writer.write_uint32("tokenizer.ggml.separator_token_id", key["id"] ) + gguf_writer.add_sep_token_id(key["id"]) if "pad_token" in tokenizer_config: for key in tokenizer["added_tokens"]: if key["content"] == tokenizer_config["pad_token"]: - gguf_writer.write_uint32("tokenizer.ggml.padding_token_id", key["id"] ) + gguf_writer.add_pad_token_id(key["id"]) # TENSORS # tensor info -print("write gguf tensor info") +print("gguf: add gguf tensor info") for name in list_vars.keys(): data = list_vars[name].squeeze().numpy() @@ -167,19 +151,25 @@ data = data.astype(np.float32) ftype_cur = 0 - gguf_writer.write_tensor_info(name, data) + gguf_writer.add_tensor_info(name, data) +print("gguf: write header") +gguf_writer.write_header_to_file() +print("gguf: write key-values") +gguf_writer.write_kv_data_to_file() +print("gguf: write tensor info") +gguf_writer.write_ti_data_to_file() # tensor data -print("write gguf tensor data") +print("gguf: write tensor data") for name in list_vars.keys(): data = list_vars[name].squeeze().numpy() - print("Process tensor: " + name + " with shape: ", data.shape) +# print("Process tensor: " + name + " with shape: ", data.shape) # we don't need these if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): - print(" Skip tensor: " + name) +# print(" Skip tensor: " + name) continue n_dims = len(data.shape) @@ -188,23 +178,23 @@ ftype_cur = 0 if ftype != 0: if name.endswith(".weight") and n_dims == 2: - print(" Converting to float16") +# print(" Converting to float16") data = data.astype(np.float16) ftype_cur = 1 else: - print(" Converting to float32") +# print(" Converting to float32") data = data.astype(np.float32) ftype_cur = 0 else: if data.dtype != np.float32: - print(" Converting to float32") +# print(" Converting to float32") data = data.astype(np.float32) ftype_cur = 0 - gguf_writer.write_tensor(data) + gguf_writer.write_tensor_to_file(data) gguf_writer.close() -print("Done. Output file: " + fname_out) +print("gguf: conversion done, output file: " + fname_out) print("") From cf365fbc2022ca25ead5568dd81b4b63349d2c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 2 Aug 2023 11:13:56 +0300 Subject: [PATCH 066/242] gguf : gguf counterpart of llama-util.h --- examples/gguf/gguf.cpp | 11 +- gguf-util.h | 451 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 455 insertions(+), 7 deletions(-) create mode 100644 gguf-util.h diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index deb4a83592779..671f8748f5e13 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -1,5 +1,5 @@ #include "ggml.h" -#include "llama-util.h" +#include "gguf-util.h" #include #include @@ -375,11 +375,8 @@ bool gguf_ex_read_2(const std::string & fname) { struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params); - // TODO: mmap based on tensor infos - - - struct llama_file file(fname.c_str(), "rb"); - llama_mmap data_mmap(&file, 0, false); + struct gguf_file file(fname.c_str(), "rb"); + gguf_mmap data_mmap(&file, 0, false); const int n_tensors = gguf_get_n_tensors(ctx); for (int i = 0; i < n_tensors; ++i) { @@ -405,7 +402,7 @@ fprintf(stdout, "%s: ctx_data size: %zu\n", __func__, ggml_get_mem_size(ctx_data ggml_free(ctx_data); gguf_free(ctx); - + return true; } diff --git a/gguf-util.h b/gguf-util.h new file mode 100644 index 0000000000000..23f411bb0246b --- /dev/null +++ b/gguf-util.h @@ -0,0 +1,451 @@ +// GGUF counterpart of llama-util.h. +// we may consider making it a part of ggml.c once GGUF work is complete. +// Contains wrappers around OS interfaces. + +#ifndef GGUF_UTIL_H +#define GGUF_UTIL_H +#include "ggml.h" +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#ifdef __has_include + #if __has_include() + #include + #if defined(_POSIX_MAPPED_FILES) + #include + #endif + #if defined(_POSIX_MEMLOCK_RANGE) + #include + #endif + #endif +#endif + +#if defined(_WIN32) + #define WIN32_LEAN_AND_MEAN + #ifndef NOMINMAX + #define NOMINMAX + #endif + #include + #include + #include // for _fseeki64 +#endif + +#ifdef __GNUC__ +#ifdef __MINGW32__ +__attribute__((format(gnu_printf, 1, 2))) +#else +__attribute__((format(printf, 1, 2))) +#endif +#endif +static std::string format(const char * fmt, ...) { + va_list ap, ap2; + va_start(ap, fmt); + va_copy(ap2, ap); + int size = vsnprintf(NULL, 0, fmt, ap); + GGML_ASSERT(size >= 0 && size < INT_MAX); + std::vector buf(size + 1); + int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2); + GGML_ASSERT(size2 == size); + va_end(ap2); + va_end(ap); + return std::string(buf.data(), size); +} + +// TODO: can we merge this one and gguf_context? +struct gguf_file { + // use FILE * so we don't have to re-open the file to mmap + FILE * fp; + size_t size; + + gguf_file(const char * fname, const char * mode) { + fp = std::fopen(fname, mode); + if (fp == NULL) { + throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); + } + seek(0, SEEK_END); + size = tell(); + seek(0, SEEK_SET); + } + + size_t tell() const { +#ifdef _WIN32 + __int64 ret = _ftelli64(fp); +#else + long ret = std::ftell(fp); +#endif + GGML_ASSERT(ret != -1); // this really shouldn't fail + return (size_t) ret; + } + + void seek(size_t offset, int whence) { +#ifdef _WIN32 + int ret = _fseeki64(fp, (__int64) offset, whence); +#else + int ret = std::fseek(fp, (long) offset, whence); +#endif + GGML_ASSERT(ret == 0); // same + } +}; + +#if defined(_WIN32) +static std::string gguf_format_win_err(DWORD err) { + LPSTR buf; + size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL); + if (!size) { + return "FormatMessageA failed"; + } + std::string ret(buf, size); + LocalFree(buf); + return ret; +} +#endif + +struct gguf_mmap { + void * addr; + size_t size; + + gguf_mmap(const gguf_mmap &) = delete; + +#ifdef _POSIX_MAPPED_FILES + static constexpr bool SUPPORTED = true; + + gguf_mmap(struct gguf_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) { + size = file->size; + int fd = fileno(file->fp); + int flags = MAP_SHARED; + // prefetch/readahead impairs performance on NUMA systems + if (numa) { prefetch = 0; } +#ifdef __linux__ + if (prefetch) { flags |= MAP_POPULATE; } +#endif + addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0); + if (addr == MAP_FAILED) { + throw std::runtime_error(format("mmap failed: %s", strerror(errno))); + } + + if (prefetch > 0) { + // Advise the kernel to preload the mapped memory + if (madvise(addr, std::min(file->size, prefetch), MADV_WILLNEED)) { + fprintf(stderr, "warning: madvise(.., MADV_WILLNEED) failed: %s\n", + strerror(errno)); + } + } + if (numa) { + // advise the kernel not to use readahead + // (because the next page might not belong on the same node) + if (madvise(addr, file->size, MADV_RANDOM)) { + fprintf(stderr, "warning: madvise(.., MADV_RANDOM) failed: %s\n", + strerror(errno)); + } + } + } + + ~gguf_mmap() { + munmap(addr, size); + } +#elif defined(_WIN32) + static constexpr bool SUPPORTED = true; + + gguf_mmap(struct llama_file * file, bool prefetch = true, bool numa = false) { + (void) numa; + + size = file->size; + + HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp)); + + HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL); + DWORD error = GetLastError(); + + if (hMapping == NULL) { + throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str())); + } + + addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); + error = GetLastError(); + CloseHandle(hMapping); + + if (addr == NULL) { + throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str())); + } + + #if _WIN32_WINNT >= _WIN32_WINNT_WIN8 + if (prefetch) { + // Advise the kernel to preload the mapped memory + WIN32_MEMORY_RANGE_ENTRY range; + range.VirtualAddress = addr; + range.NumberOfBytes = (SIZE_T)size; + if (!PrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) { + fprintf(stderr, "warning: PrefetchVirtualMemory failed: %s\n", + gguf_format_win_err(GetLastError()).c_str()); + } + } + #else + #pragma message("warning: You are building for pre-Windows 8; prefetch not supported") + #endif // _WIN32_WINNT >= _WIN32_WINNT_WIN8 + } + + ~gguf_mmap() { + if (!UnmapViewOfFile(addr)) { + fprintf(stderr, "warning: UnmapViewOfFile failed: %s\n", + llama_format_win_err(GetLastError()).c_str()); + } + } +#else + static constexpr bool SUPPORTED = false; + + gguf_mmap(struct llama_file *, bool prefetch = true, bool numa = false) { + (void) prefetch; + (void) numa; + + throw std::runtime_error(std::string("mmap not supported")); + } +#endif +}; + +// Represents some region of memory being locked using mlock or VirtualLock; +// will automatically unlock on destruction. +struct gguf_mlock { + void * addr = NULL; + size_t size = 0; + bool failed_already = false; + + gguf_mlock() {} + gguf_mlock(const gguf_mlock &) = delete; + + ~gguf_mlock() { + if (size) { + raw_unlock(addr, size); + } + } + + void init(void * ptr) { + GGML_ASSERT(addr == NULL && size == 0); + addr = ptr; + } + + void grow_to(size_t target_size) { + GGML_ASSERT(addr); + if (failed_already) { + return; + } + size_t granularity = lock_granularity(); + target_size = (target_size + granularity - 1) & ~(granularity - 1); + if (target_size > size) { + if (raw_lock((uint8_t *) addr + size, target_size - size)) { + size = target_size; + } else { + failed_already = true; + } + } + } + +#ifdef _POSIX_MEMLOCK_RANGE + static constexpr bool SUPPORTED = true; + + size_t lock_granularity() { + return (size_t) sysconf(_SC_PAGESIZE); + } + + #ifdef __APPLE__ + #define MLOCK_SUGGESTION \ + "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \ + "decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MLOCK (ulimit -l).\n" + #else + #define MLOCK_SUGGESTION \ + "Try increasing RLIMIT_MLOCK ('ulimit -l' as root).\n" + #endif + + bool raw_lock(const void * addr, size_t size) { + if (!mlock(addr, size)) { + return true; + } else { + char* errmsg = std::strerror(errno); + bool suggest = (errno == ENOMEM); + + // Check if the resource limit is fine after all + struct rlimit lock_limit; + if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) + suggest = false; + if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size)) + suggest = false; + + fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s", + size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : ""); + return false; + } + } + + #undef MLOCK_SUGGESTION + + void raw_unlock(void * addr, size_t size) { + if (munlock(addr, size)) { + fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno)); + } + } +#elif defined(_WIN32) + static constexpr bool SUPPORTED = true; + + size_t lock_granularity() { + SYSTEM_INFO si; + GetSystemInfo(&si); + return (size_t) si.dwPageSize; + } + + bool raw_lock(void * ptr, size_t len) { + for (int tries = 1; ; tries++) { + if (VirtualLock(ptr, len)) { + return true; + } + if (tries == 2) { + fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n", + len, size, llama_format_win_err(GetLastError()).c_str()); + return false; + } + + // It failed but this was only the first try; increase the working + // set size and try again. + SIZE_T min_ws_size, max_ws_size; + if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) { + fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n", + gguf_format_win_err(GetLastError()).c_str()); + return false; + } + // Per MSDN: "The maximum number of pages that a process can lock + // is equal to the number of pages in its minimum working set minus + // a small overhead." + // Hopefully a megabyte is enough overhead: + size_t increment = len + 1048576; + // The minimum must be <= the maximum, so we need to increase both: + min_ws_size += increment; + max_ws_size += increment; + if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) { + fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n", + gguf_format_win_err(GetLastError()).c_str()); + return false; + } + } + } + + void raw_unlock(void * ptr, size_t len) { + if (!VirtualUnlock(ptr, len)) { + fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n", + gguf_format_win_err(GetLastError()).c_str()); + } + } +#else + static constexpr bool SUPPORTED = false; + + size_t lock_granularity() { + return (size_t) 65536; + } + + bool raw_lock(const void * addr, size_t len) { + fprintf(stderr, "warning: mlock not supported on this system\n"); + return false; + } + + void raw_unlock(const void * addr, size_t len) {} +#endif +}; + +// Replacement for std::vector that doesn't require zero-initialization. +struct gguf_buffer { + uint8_t * addr = NULL; + size_t size = 0; + + gguf_buffer() = default; + + void resize(size_t len) { +#ifdef GGML_USE_METAL + free(addr); + int result = posix_memalign((void **) &addr, getpagesize(), len); + if (result == 0) { + memset(addr, 0, len); + } + else { + addr = NULL; + } +#else + delete[] addr; + addr = new uint8_t[len]; +#endif + size = len; + } + + ~gguf_buffer() { +#ifdef GGML_USE_METAL + free(addr); +#else + delete[] addr; +#endif + addr = NULL; + } + + // disable copy and move + gguf_buffer(const gguf_buffer&) = delete; + gguf_buffer(gguf_buffer&&) = delete; + gguf_buffer& operator=(const gguf_buffer&) = delete; + gguf_buffer& operator=(gguf_buffer&&) = delete; +}; + +#ifdef GGML_USE_CUBLAS +#include "ggml-cuda.h" +struct gguf_ctx_buffer { + uint8_t * addr = NULL; + bool is_cuda; + size_t size = 0; + + gguf_ctx_buffer() = default; + + void resize(size_t size) { + free(); + + addr = (uint8_t *) ggml_cuda_host_malloc(size); + if (addr) { + is_cuda = true; + } + else { + // fall back to pageable memory + addr = new uint8_t[size]; + is_cuda = false; + } + this->size = size; + } + + void free() { + if (addr) { + if (is_cuda) { + ggml_cuda_host_free(addr); + } + else { + delete[] addr; + } + } + addr = NULL; + } + + ~gguf_ctx_buffer() { + free(); + } + + // disable copy and move + gguf_ctx_buffer(const gguf_ctx_buffer&) = delete; + gguf_ctx_buffer(gguf_ctx_buffer&&) = delete; + gguf_ctx_buffer& operator=(const gguf_ctx_buffer&) = delete; + gguf_ctx_buffer& operator=(gguf_ctx_buffer&&) = delete; +}; +#else +typedef gguf_buffer gguf_ctx_buffer; +#endif + +#endif From c3a65c4bbe1cd64cd6255e70b5c7563fb977959f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Wed, 2 Aug 2023 11:16:23 +0300 Subject: [PATCH 067/242] gguf-util.h : update note --- gguf-util.h | 1 + 1 file changed, 1 insertion(+) diff --git a/gguf-util.h b/gguf-util.h index 23f411bb0246b..74d6e61f792d0 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -1,5 +1,6 @@ // GGUF counterpart of llama-util.h. // we may consider making it a part of ggml.c once GGUF work is complete. +// this will require extra work to migrate this to pure C. // Contains wrappers around OS interfaces. #ifndef GGUF_UTIL_H From e1e9b285477670d0ee69474f3e5b339a5b4741bf Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Wed, 2 Aug 2023 11:15:33 +0200 Subject: [PATCH 068/242] convert-llama-h5-to-gguf.py : accumulate kv / ti + special tokens --- convert-llama-h5-to-gguf.py | 120 +++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 44 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 7f16905959b73..7a5dae77a249b 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -11,8 +11,10 @@ from sentencepiece import SentencePieceProcessor -NDArray = np.ndarray[Any, Any] +#NDArray = np.ndarray[Any, Any] +# compatible with python < 3.9 +NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]' def permute(weights: NDArray, n_head: int) -> NDArray: return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) @@ -57,51 +59,36 @@ def permute(weights: NDArray, n_head: int) -> NDArray: model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) list_vars = model.state_dict() -# count tensors to be converted -tensor_count = 0 -for name in list_vars.keys(): - # we don't need these - if name.endswith(".rotary_emb.inv_freq"): - continue - tensor_count += 1 - gguf_writer = gguf.GGUFWriter.open(fname_out) -# This must be changed when adding/deleting kv -kv_count = 13 -print("tensors " + str(tensor_count) + " kv " + str(kv_count)) - -print("write gguf header") - -gguf_writer.write_header(tensor_count, kv_count) - -print("write gguf hparams") +print("gguf: add key-values, metadata") llm_arch = "llama" -gguf_writer.write_name("llama2-7b") -gguf_writer.write_description("gguf test model") -gguf_writer.write_architecture(llm_arch) -gguf_writer.write_context_length(llm_arch, hparams["max_position_embeddings"]) -gguf_writer.write_embedding_length(llm_arch, hparams["hidden_size"]) -gguf_writer.write_layer_count(llm_arch, hparams["num_hidden_layers"]) -gguf_writer.write_feed_forward_length(llm_arch, hparams["intermediate_size"]) -gguf_writer.write_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"]) -gguf_writer.write_head_count(llm_arch, hparams["num_attention_heads"]) -gguf_writer.write_layer_norm_rms_eps(llm_arch, hparams["rms_norm_eps"]) +gguf_writer.add_name("llama2-7b") +gguf_writer.add_description("gguf test model") +gguf_writer.add_architecture(llm_arch) +gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) +gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) +gguf_writer.add_layer_count(llm_arch, hparams["num_hidden_layers"]) +gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"]) +gguf_writer.add_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"]) +gguf_writer.add_head_count(llm_arch, hparams["num_attention_heads"]) +gguf_writer.add_layer_norm_rms_eps(llm_arch, hparams["rms_norm_eps"]) # TOKENIZATION -print("write gguf tokenizer") +print("gguf: add key-values, tokenizer") tokens: List[str] = [] scores: List[float] = [] if Path(dir_model + "/tokenizer.model").is_file(): # vocab type sentencepiece - print("Adding sentencepiece tokenizer vocab.") + print("gguf: adding sentencepiece tokenizer vocab") + tokenizer = SentencePieceProcessor(dir_model + "/tokenizer.model") for i in range(tokenizer.vocab_size()): @@ -123,14 +110,52 @@ def permute(weights: NDArray, n_head: int) -> NDArray: tokens.append(text) scores.append(score) -gguf_writer.write_tokenizer_model("llama") -gguf_writer.write_token_list(tokens) -gguf_writer.write_token_scores(scores) + gguf_writer.add_tokenizer_model("llama") + gguf_writer.add_token_list(tokens) + gguf_writer.add_token_scores(scores) + +if Path(dir_model + "/tokenizer.json").is_file(): + with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f: + tokenizer = json.load(f) + + if "added_tokens" in tokenizer and Path(dir_model + "/tokenizer_config.json").is_file(): + print("gguf: adding special token ids") + + with open(dir_model + "/tokenizer_config.json", "r", encoding="utf-8") as f: + tokenizer_config = json.load(f) + + # find special token ids + + if "bos_token" in tokenizer_config and tokenizer_config["bos_token"] != None: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["bos_token"] or key["content"] == tokenizer_config["bos_token"]["content"]: + gguf_writer.add_bos_token_id(key["id"]) + + if "eos_token" in tokenizer_config and tokenizer_config["eos_token"] != None: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["eos_token"] or key["content"] == tokenizer_config["eos_token"]["content"]: + gguf_writer.add_eos_token_id(key["id"]) + + if "unk_token" in tokenizer_config and tokenizer_config["unk_token"] != None: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["unk_token"] or key["content"] == tokenizer_config["unk_token"]["content"]: + gguf_writer.add_unk_token_id(key["id"]) + + if "sep_token" in tokenizer_config and tokenizer_config["sep_token"] != None: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["sep_token"] or key["content"] == tokenizer_config["sep_token"]["content"]: + gguf_writer.add_sep_token_id(key["id"]) + + if "pad_token" in tokenizer_config and tokenizer_config["pad_token"] != None: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["pad_token"] or key["content"] == tokenizer_config["pad_token"]["content"]: + gguf_writer.add_pad_token_id(key["id"]) + # TENSORS # tensor info -print("write gguf tensor info") +print("gguf: add gguf tensor info") for name in list_vars.keys(): data = list_vars[name].squeeze().numpy() @@ -197,24 +222,31 @@ def permute(weights: NDArray, n_head: int) -> NDArray: data = data.astype(np.float32) ftype_cur = 0 - gguf_writer.write_tensor_info(name, data) + gguf_writer.add_tensor_info(name, data) + +print("gguf: write header") +gguf_writer.write_header_to_file() +print("gguf: write key-values") +gguf_writer.write_kv_data_to_file() +print("gguf: write tensor info") +gguf_writer.write_ti_data_to_file() # tensor data -print("write gguf tensor data") +print("gguf: write tensor data") for name in list_vars.keys(): data = list_vars[name].squeeze().numpy() - print("Process tensor: " + name + " with shape: ", data.shape) +# print("Process tensor: " + name + " with shape: ", data.shape) # we don't need these if name.endswith(".rotary_emb.inv_freq"): - print(" Skip tensor: " + name) +# print(" Skip tensor: " + name) continue # permute these if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): - print(" Permute tensor: " + name) +# print(" Permute tensor: " + name) data = permute(data, hparams["num_attention_heads"]) n_dims = len(data.shape) @@ -223,23 +255,23 @@ def permute(weights: NDArray, n_head: int) -> NDArray: ftype_cur = 0 if ftype != 0: if name.endswith(".weight") and n_dims == 2: - print(" Converting to float16") +# print(" Converting to float16") data = data.astype(np.float16) ftype_cur = 1 else: - print(" Converting to float32") +# print(" Converting to float32") data = data.astype(np.float32) ftype_cur = 0 else: if data.dtype != np.float32: - print(" Converting to float32") +# print(" Converting to float32") data = data.astype(np.float32) ftype_cur = 0 - gguf_writer.write_tensor(data) + gguf_writer.write_tensor_to_file(data) gguf_writer.close() -print("Done. Output file: " + fname_out) +print("gguf: conversion done, output file: " + fname_out) print("") From c5ba5efda2a3b3b9ee3d42d908dde2b92dc80bc1 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Wed, 2 Aug 2023 11:26:07 +0200 Subject: [PATCH 069/242] convert-llama-h5-to-gguf.py : special tokens --- convert-llama-h5-to-gguf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 7a5dae77a249b..67b3c55d4e35f 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -128,27 +128,27 @@ def permute(weights: NDArray, n_head: int) -> NDArray: if "bos_token" in tokenizer_config and tokenizer_config["bos_token"] != None: for key in tokenizer["added_tokens"]: - if key["content"] == tokenizer_config["bos_token"] or key["content"] == tokenizer_config["bos_token"]["content"]: + if key["content"] == tokenizer_config["bos_token"]["content"]: gguf_writer.add_bos_token_id(key["id"]) if "eos_token" in tokenizer_config and tokenizer_config["eos_token"] != None: for key in tokenizer["added_tokens"]: - if key["content"] == tokenizer_config["eos_token"] or key["content"] == tokenizer_config["eos_token"]["content"]: + if key["content"] == tokenizer_config["eos_token"]["content"]: gguf_writer.add_eos_token_id(key["id"]) if "unk_token" in tokenizer_config and tokenizer_config["unk_token"] != None: for key in tokenizer["added_tokens"]: - if key["content"] == tokenizer_config["unk_token"] or key["content"] == tokenizer_config["unk_token"]["content"]: + if key["content"] == tokenizer_config["unk_token"]["content"]: gguf_writer.add_unk_token_id(key["id"]) if "sep_token" in tokenizer_config and tokenizer_config["sep_token"] != None: for key in tokenizer["added_tokens"]: - if key["content"] == tokenizer_config["sep_token"] or key["content"] == tokenizer_config["sep_token"]["content"]: + if key["content"] == tokenizer_config["sep_token"]["content"]: gguf_writer.add_sep_token_id(key["id"]) if "pad_token" in tokenizer_config and tokenizer_config["pad_token"] != None: for key in tokenizer["added_tokens"]: - if key["content"] == tokenizer_config["pad_token"] or key["content"] == tokenizer_config["pad_token"]["content"]: + if key["content"] == tokenizer_config["pad_token"]["content"]: gguf_writer.add_pad_token_id(key["id"]) From 23abbe8e00b14d529778195c404005f0740e6d6b Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 4 Aug 2023 03:51:43 +0200 Subject: [PATCH 070/242] Delete gptneox-common.cpp --- gptneox-common.cpp | 601 --------------------------------------------- 1 file changed, 601 deletions(-) delete mode 100644 gptneox-common.cpp diff --git a/gptneox-common.cpp b/gptneox-common.cpp deleted file mode 100644 index 9dee0cb9ce8b2..0000000000000 --- a/gptneox-common.cpp +++ /dev/null @@ -1,601 +0,0 @@ -#include "gptneox-common.h" - -#include -#include -#include -#include -#include -#include -#include - -#if defined(_MSC_VER) -#pragma warning(disable: 4244 4267) // possible loss of data -#endif - -// Function to check if the next argument exists -std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) { - if (i + 1 < argc && argv[i + 1][0] != '-') { - return argv[++i]; - } else { - fprintf(stderr, "error: %s requires one argument.\n", flag.c_str()); - gpt_print_usage(argc, argv, params); - exit(0); - } -} - -bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { - for (int i = 1; i < argc; i++) { - std::string arg = argv[i]; - - if (arg == "-s" || arg == "--seed") { - params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-t" || arg == "--threads") { - params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") { - params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-p" || arg == "--prompt") { - params.prompt = get_next_arg(i, argc, argv, arg, params); - } else if (arg == "-n" || arg == "--n_predict") { - params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--top_k") { - params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--top_p") { - params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--temp") { - params.temp = std::stof(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--repeat-last-n") { - params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "--repeat-penalty") { - params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-b" || arg == "--batch_size") { - params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-m" || arg == "--model") { - params.model = get_next_arg(i, argc, argv, arg, params); - } else if (arg == "-i" || arg == "--interactive") { - params.interactive = true; - } else if (arg == "-ip" || arg == "--interactive-port") { - params.interactive = true; - params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params)); - } else if (arg == "-h" || arg == "--help") { - gpt_print_usage(argc, argv, params); - exit(0); - } else if (arg == "-f" || arg == "--file") { - get_next_arg(i, argc, argv, arg, params); - std::ifstream file(argv[i]); - if (!file) { - fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); - break; - } - std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.prompt)); - if (params.prompt.back() == '\n') { - params.prompt.pop_back(); - } - } else if (arg == "-tt" || arg == "--token_test") { - params.token_test = get_next_arg(i, argc, argv, arg, params); - } - else { - fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); - gpt_print_usage(argc, argv, params); - exit(0); - } - } - - return true; -} - -void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { - fprintf(stderr, "usage: %s [options]\n", argv[0]); - fprintf(stderr, "\n"); - fprintf(stderr, "options:\n"); - fprintf(stderr, " -h, --help show this help message and exit\n"); - fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n"); - fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); - fprintf(stderr, " -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers); - fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); - fprintf(stderr, " prompt to start generation with (default: random)\n"); - fprintf(stderr, " -f FNAME, --file FNAME\n"); - fprintf(stderr, " load prompt from a file\n"); - fprintf(stderr, " -tt TOKEN_TEST, --token_test TOKEN_TEST\n"); - fprintf(stderr, " test tokenization\n"); - fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d)\n", params.n_predict); - fprintf(stderr, " --top_k N top-k sampling (default: %d)\n", params.top_k); - fprintf(stderr, " --top_p N top-p sampling (default: %.1f)\n", params.top_p); - fprintf(stderr, " --temp N temperature (default: %.1f)\n", params.temp); - fprintf(stderr, " --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled)\n", params.repeat_last_n); - fprintf(stderr, " --repeat-penalty N penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\n", (double)params.repeat_penalty); - fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch); - fprintf(stderr, " -m FNAME, --model FNAME\n"); - fprintf(stderr, " model path (default: %s)\n", params.model.c_str()); - fprintf(stderr, "\n"); -} - -std::string gpt_random_prompt(std::mt19937 & rng) { - const int r = rng() % 10; - switch (r) { - case 0: return "So"; - case 1: return "Once upon a time"; - case 2: return "When"; - case 3: return "The"; - case 4: return "After"; - case 5: return "If"; - case 6: return "import"; - case 7: return "He"; - case 8: return "She"; - case 9: return "They"; - default: return "To"; - } - - return "The"; -} - -std::string trim(const std::string & s) { - std::regex e("^\\s+|\\s+$"); - return std::regex_replace(s, e, ""); -} - -std::string replace(const std::string & s, const std::string & from, const std::string & to) { - std::string result = s; - size_t pos = 0; - while ((pos = result.find(from, pos)) != std::string::npos) { - result.replace(pos, from.length(), to); - pos += to.length(); - } - return result; -} - -void gpt_vocab::add_special_token(const std::string & token) { - special_tokens.push_back(token); -} - -std::map json_parse(const std::string & fname) { - std::map result; - - // read file into string - std::string json; - { - std::ifstream ifs(fname); - if (!ifs) { - fprintf(stderr, "Failed to open %s\n", fname.c_str()); - exit(1); - } - - json = std::string((std::istreambuf_iterator(ifs)), - (std::istreambuf_iterator())); - } - - if (json[0] != '{') { - return result; - } - - // parse json - { - bool has_key = false; - bool in_token = false; - - std::string str_key = ""; - std::string str_val = ""; - - int n = json.size(); - for (int i = 1; i < n; ++i) { - if (!in_token) { - if (json[i] == ' ') continue; - if (json[i] == '"') { - in_token = true; - continue; - } - } else { - if (json[i] == '\\' && i+1 < n) { - if (has_key == false) { - str_key += json[i]; - } else { - str_val += json[i]; - } - ++i; - } else if (json[i] == '"') { - if (has_key == false) { - has_key = true; - ++i; - while (json[i] == ' ') ++i; - ++i; // : - while (json[i] == ' ') ++i; - if (json[i] != '\"') { - while (json[i] != ',' && json[i] != '}') { - str_val += json[i++]; - } - has_key = false; - } else { - in_token = true; - continue; - } - } else { - has_key = false; - } - - str_key = ::replace(str_key, "\\u0120", " " ); // \u0120 -> space - str_key = ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line - str_key = ::replace(str_key, "\\\"", "\""); // \\\" -> " - - try { - result[str_key] = std::stoi(str_val); - } catch (...) { - //fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str()); - - } - str_key = ""; - str_val = ""; - in_token = false; - continue; - } - if (has_key == false) { - str_key += json[i]; - } else { - str_val += json[i]; - } - } - } - } - - return result; -} - -std::string convert_to_utf8(const std::wstring & input) { - std::wstring_convert> converter; - return converter.to_bytes(input); -} - - -std::wstring convert_to_wstring(const std::string & input) { - std::wstring_convert> converter; - return converter.from_bytes(input); -} - -void gpt_split_words(std::string str, std::vector& words) { - const std::string pattern = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"; - const std::regex re(pattern); - std::smatch m; - - while (std::regex_search(str, m, re)) { - for (auto x : m) { - words.push_back(x); - } - str = m.suffix(); - } -} - -std::vector gpt_tokenize(const gpt_vocab & vocab, const std::string & text) { - std::vector words; - - // first split the text into words - { - std::string str = text; - - // Generate the subpattern from the special_tokens vector if it's not empty - if (!vocab.special_tokens.empty()) { - const std::regex escape(R"([\[\\\^\$\.\|\?\*\+\(\)\{\}])"); - std::string special_tokens_subpattern; - for (const auto & token : vocab.special_tokens) { - if (!special_tokens_subpattern.empty()) { - special_tokens_subpattern += "|"; - } - special_tokens_subpattern += std::regex_replace(token, escape, R"(\$&)"); - } - - std::regex re(special_tokens_subpattern); - std::smatch m; - // Split the text by special tokens. - while (std::regex_search(str, m, re)) { - // Split the substrings in-between special tokens into words. - gpt_split_words(m.prefix(), words); - // Add matched special tokens as words. - for (auto x : m) { - words.push_back(x); - } - str = m.suffix(); - } - // Remaining text without special tokens will be handled below. - } - - gpt_split_words(str, words); - } - - // find the longest token that forms each word in words: - std::vector tokens; - for (const auto & word : words) { - for (int i = 0; i < (int) word.size(); ){ - for (int j = word.size() - 1; j >= i; j--){ - auto cand = word.substr(i, j-i+1); - auto it = vocab.token_to_id.find(cand); - if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab - tokens.push_back(it->second); - i = j + 1; - break; - } - else if (j == i){ // word.substr(i, 1) has no matching - fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data()); - i++; - } - } - } - } - - return tokens; -} - -std::vector parse_tokens_from_string(const std::string& input, char delimiter) { - std::vector output; - std::stringstream ss(input); - std::string token; - - while (std::getline(ss, token, delimiter)) { - output.push_back(std::stoi(token)); - } - - return output; -} - -std::map> extract_tests_from_file(const std::string & fpath_test){ - if (fpath_test.empty()){ - fprintf(stderr, "%s : No test file found.\n", __func__); - return std::map>(); - } - - std::map> tests; - - auto fin = std::ifstream(fpath_test, std::ios_base::in); - const char * delimeter = " => "; - const char del_tok = ','; - std::string line; - while (std::getline(fin, line)) { - size_t delimiterPos = line.find(delimeter); - if (delimiterPos != std::string::npos) { - std::string text = line.substr(0, delimiterPos); - std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter)); - tests[text] = parse_tokens_from_string(s_tokens, del_tok); - } - } - return tests; -} - -void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){ - std::map> tests = extract_tests_from_file(fpath_test); - - size_t n_fails = 0; - - for (const auto & test : tests) { - std::vector tokens = gpt_tokenize(vocab, test.first); - - if (tokens != test.second){ - n_fails++; - - // print out failure cases - fprintf(stderr, "%s : failed test: '%s'\n", __func__, test.first.c_str()); - fprintf(stderr, "%s : tokens in hf: ", __func__); - for (const auto & t : test.second) { - fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t); - } - fprintf(stderr, "\n"); - fprintf(stderr, "%s : tokens in ggml: ", __func__); - for (const auto & t : tokens) { - fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t); - } - fprintf(stderr, "\n"); - } - } - - fprintf(stderr, "%s : %zu tests failed out of %zu tests.\n", __func__, n_fails, tests.size()); -} - -bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) { - printf("%s: loading vocab from '%s'\n", __func__, fname.c_str()); - - vocab.token_to_id = ::json_parse(fname); - - for (const auto & kv : vocab.token_to_id) { - vocab.id_to_token[kv.second] = kv.first; - } - - printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size()); - - // print the vocabulary - //for (auto kv : vocab.token_to_id) { - // printf("'%s' -> %d\n", kv.first.data(), kv.second); - //} - - return true; -} - -gpt_vocab::id gpt_sample_top_k_top_p( - const gpt_vocab & vocab, - const float * logits, - int top_k, - double top_p, - double temp, - std::mt19937 & rng) { - int n_logits = vocab.id_to_token.size(); - - std::vector> logits_id; - logits_id.reserve(n_logits); - - { - const double scale = 1.0/temp; - for (int i = 0; i < n_logits; ++i) { - logits_id.push_back(std::make_pair(logits[i]*scale, i)); - } - } - - // find the top K tokens - std::partial_sort( - logits_id.begin(), - logits_id.begin() + top_k, logits_id.end(), - [](const std::pair & a, const std::pair & b) { - return a.first > b.first; - }); - - logits_id.resize(top_k); - - double maxl = -INFINITY; - for (const auto & kv : logits_id) { - maxl = std::max(maxl, kv.first); - } - - // compute probs for the top K tokens - std::vector probs; - probs.reserve(logits_id.size()); - - double sum = 0.0; - for (const auto & kv : logits_id) { - double p = exp(kv.first - maxl); - probs.push_back(p); - sum += p; - } - - // normalize the probs - for (auto & p : probs) { - p /= sum; - } - - if (top_p < 1.0f) { - double cumsum = 0.0f; - for (int i = 0; i < top_k; i++) { - cumsum += probs[i]; - if (cumsum >= top_p) { - top_k = i + 1; - probs.resize(top_k); - logits_id.resize(top_k); - break; - } - } - - cumsum = 1.0/cumsum; - for (int i = 0; i < (int) probs.size(); i++) { - probs[i] *= cumsum; - } - } - - //printf("\n"); - //for (int i = 0; i < (int) probs.size(); i++) { - // printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]); - //} - //exit(0); - - std::discrete_distribution<> dist(probs.begin(), probs.end()); - int idx = dist(rng); - - return logits_id[idx].second; -} - -gpt_vocab::id gpt_sample_top_k_top_p_repeat( - const gpt_vocab & vocab, - const float * logits, - const int32_t * last_n_tokens_data, - size_t last_n_tokens_data_size, - int top_k, - double top_p, - double temp, - int repeat_last_n, - float repeat_penalty, - std::mt19937 & rng) { - - int n_logits = vocab.id_to_token.size(); - - const auto * plogits = logits; - - const auto last_n_tokens = std::vector(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size); - - if (temp <= 0) { - // select the token with the highest logit directly - float max_logit = plogits[0]; - gpt_vocab::id max_id = 0; - - for (int i = 1; i < n_logits; ++i) { - if (plogits[i] > max_logit) { - max_logit = plogits[i]; - max_id = i; - } - } - return max_id; - } - - - std::vector> logits_id; - logits_id.reserve(n_logits); - - { - const float scale = 1.0f/temp; - for (int i = 0; i < n_logits; ++i) { - // repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858) - // credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main - if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) { - // if score < 0 then repetition penalty has to multiplied to reduce the previous token probability - if (plogits[i] < 0.0f) { - logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i)); - } else { - logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i)); - } - } else { - logits_id.push_back(std::make_pair(plogits[i]*scale, i)); - } - } - } - - // find the top K tokens - std::partial_sort( - logits_id.begin(), - logits_id.begin() + top_k, logits_id.end(), - [](const std::pair & a, const std::pair & b) { - return a.first > b.first; - }); - - logits_id.resize(top_k); - - double maxl = -INFINITY; - for (const auto & kv : logits_id) { - maxl = std::max(maxl, kv.first); - } - - // compute probs for the top K tokens - std::vector probs; - probs.reserve(logits_id.size()); - - double sum = 0.0; - for (const auto & kv : logits_id) { - double p = exp(kv.first - maxl); - probs.push_back(p); - sum += p; - } - - // normalize the probs - for (auto & p : probs) { - p /= sum; - } - - if (top_p < 1.0f) { - double cumsum = 0.0f; - for (int i = 0; i < top_k; i++) { - cumsum += probs[i]; - if (cumsum >= top_p) { - top_k = i + 1; - probs.resize(top_k); - logits_id.resize(top_k); - break; - } - } - - cumsum = 1.0/cumsum; - for (int i = 0; i < (int) probs.size(); i++) { - probs[i] *= cumsum; - } - } - -// printf("\n"); -// for (int i = 0; i < (int) probs.size(); i++) { -// for (int i = 0; i < 10; i++) { -// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]); -// } - - std::discrete_distribution<> dist(probs.begin(), probs.end()); - int idx = dist(rng); - - return logits_id[idx].second; - -} From 6691aa879755d4142a88a62466856b31998ad0d4 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 4 Aug 2023 03:52:01 +0200 Subject: [PATCH 071/242] Delete gptneox-common.h --- gptneox-common.h | 125 ----------------------------------------------- 1 file changed, 125 deletions(-) delete mode 100644 gptneox-common.h diff --git a/gptneox-common.h b/gptneox-common.h deleted file mode 100644 index 60e5650c129ab..0000000000000 --- a/gptneox-common.h +++ /dev/null @@ -1,125 +0,0 @@ -// Various helper functions and utilities - -#pragma once - -#include -#include -#include -#include -#include - -// -// CLI argument parsing -// - -struct gpt_params { - int32_t seed = -1; // RNG seed - int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); - int32_t n_predict = 200; // new tokens to predict - int32_t n_batch = 8; // batch size for prompt processing - - // sampling parameters - int32_t top_k = 40; - float top_p = 0.9f; - float temp = 0.9f; - int32_t repeat_last_n = 64; - float repeat_penalty = 1.00f; - - std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path - std::string prompt = ""; - std::string token_test = ""; - - bool interactive = false; - int32_t interactive_port = -1; - - int32_t n_gpu_layers = 0; -}; - -bool gpt_params_parse(int argc, char ** argv, gpt_params & params); - -void gpt_print_usage(int argc, char ** argv, const gpt_params & params); - -std::string gpt_random_prompt(std::mt19937 & rng); - -// -// Vocab utils -// - -std::string trim(const std::string & s); - -std::string replace( - const std::string & s, - const std::string & from, - const std::string & to); - -struct gpt_vocab { - using id = int32_t; - using token = std::string; - - std::map token_to_id; - std::map id_to_token; - std::vector special_tokens; - - void add_special_token(const std::string & token); -}; - -// poor-man's JSON parsing -std::map json_parse(const std::string & fname); - -std::string convert_to_utf8(const std::wstring & input); - -std::wstring convert_to_wstring(const std::string & input); - -void gpt_split_words(std::string str, std::vector& words); - -// split text into tokens -// -// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53 -// -// Regex (Python): -// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" -// -// Regex (C++): -// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)" -// -std::vector gpt_tokenize(const gpt_vocab & vocab, const std::string & text); - -// test outputs of gpt_tokenize -// -// - compare with tokens generated by the huggingface tokenizer -// - test cases are chosen based on the model's main language (under 'prompt' directory) -// - if all sentences are tokenized identically, print 'All tests passed.' -// - otherwise, print sentence, huggingface tokens, ggml tokens -// -void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test); - -// load the tokens from encoder.json -bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab); - -// sample next token given probabilities for each embedding -// -// - consider only the top K tokens -// - from them, consider only the top tokens with cumulative probability > P -// -// TODO: not sure if this implementation is correct -// TODO: temperature is not implemented -// -gpt_vocab::id gpt_sample_top_k_top_p( - const gpt_vocab & vocab, - const float * logits, - int top_k, - double top_p, - double temp, - std::mt19937 & rng); - -gpt_vocab::id gpt_sample_top_k_top_p_repeat( - const gpt_vocab & vocab, - const float * logits, - const int32_t * last_n_tokens_data, - size_t last_n_tokens_data_size, - int top_k, - double top_p, - double temp, - int repeat_last_n, - float repeat_penalty, - std::mt19937 & rng); From 2922280a1a6615e66902ecaf2f1dddc7e605b94a Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 4 Aug 2023 03:55:23 +0200 Subject: [PATCH 072/242] convert-gptneox-h5-to-gguf.py : gpt2bpe tokenizer --- convert-gptneox-h5-to-gguf.py | 98 +++++++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 26 deletions(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index 4f9d41487d22f..066ed0da07697 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -1,14 +1,36 @@ # Quick and dirty HF gptneox--> gguf conversion import gguf +import os import sys import struct import json import numpy as np from typing import Any, List from pathlib import Path -from transformers import AutoModelForCausalLM - +from transformers import AutoTokenizer, AutoModelForCausalLM + +# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a significant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) if len(sys.argv) < 3: print("Usage: convert-h5-to-ggml.py dir-model ftype\n") @@ -20,7 +42,7 @@ # output in the same directory as the model dir_model = sys.argv[1] fname_out = sys.argv[1] + "/ggml-model.bin" - +last_dir = os.path.basename(os.path.normpath(dir_model)) # possible tensor data types # ftype == 0 -> float32 @@ -37,6 +59,8 @@ sys.exit(1) fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" +print("gguf: loading model "+last_dir) + with open(dir_model + "/config.json", "r", encoding="utf-8") as f: hparams = json.load(f) @@ -44,17 +68,17 @@ print("Model architecture not supported: " + hparams["architectures"][0] ) sys.exit() + model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) list_vars = model.state_dict() gguf_writer = gguf.GGUFWriter.open(fname_out) - -print("gguf: add key-values, metadata") +print("gguf: add metadata") llm_arch = "gptneox" -gguf_writer.add_name("pythia-70b-deduped") +gguf_writer.add_name(last_dir) gguf_writer.add_description("gguf test model") gguf_writer.add_architecture(llm_arch) gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) @@ -68,28 +92,55 @@ # TOKENIZATION -print("gguf: add key-values, tokenizer") +print("gguf: add tokenizer") tokens: List[str] = [] merges: List[str] = [] + if Path(dir_model + "/tokenizer.json").is_file(): - # vocab type gpt2 - print("gguf: adding gpt2 tokenizer vocab") + # gpt2 tokenizer + gguf_writer.add_tokenizer_model("gpt2") + + print("gguf: adding gpt2 tokenizer merges") with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f: - tokenizer = json.load(f) + tokenizer_json = json.load(f) + merges = tokenizer_json["model"]["merges"] - for key in tokenizer["model"]["vocab"]: - tokens.append(key) + gguf_writer.add_token_merges(merges) - merges = tokenizer["model"]["merges"] + print("gguf: adding gpt2 tokenizer vocab") + + vocab_size = len( tokenizer_json["model"]["vocab"] ) + + # from ggllm.cpp falcon_convert.py + tokenizer = AutoTokenizer.from_pretrained(dir_model) + + reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()} + byte_encoder = bytes_to_unicode() + byte_decoder = {v:k for k, v in byte_encoder.items()} + + for i in range(vocab_size): + if i in reverse_vocab: + try: + text = bytearray([byte_decoder[c] for c in reverse_vocab[i]]) + except KeyError: + text = bytearray() + for c in reverse_vocab[i]: + if ord(c) < 256: # single byte character + text.append(byte_decoder[ord(c)]) + else: # multibyte special token character + text.extend(c.encode('utf-8')) + else: + print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.") + padding_token = f"[PAD{i}]".encode("utf8") + text = bytearray(padding_token) + tokens.append(text) - gguf_writer.add_tokenizer_model("gpt2") gguf_writer.add_token_list(tokens) - gguf_writer.add_token_merges(merges) - if "added_tokens" in tokenizer and Path(dir_model + "/tokenizer_config.json").is_file(): + if "added_tokens" in tokenizer_json and Path(dir_model + "/tokenizer_config.json").is_file(): print("gguf: adding special token ids") with open(dir_model + "/tokenizer_config.json", "r", encoding="utf-8") as f: @@ -98,27 +149,27 @@ # find special token ids if "bos_token" in tokenizer_config: - for key in tokenizer["added_tokens"]: + for key in tokenizer_json["added_tokens"]: if key["content"] == tokenizer_config["bos_token"]: gguf_writer.add_bos_token_id(key["id"]) if "eos_token" in tokenizer_config: - for key in tokenizer["added_tokens"]: + for key in tokenizer_json["added_tokens"]: if key["content"] == tokenizer_config["eos_token"]: gguf_writer.add_eos_token_id(key["id"]) if "unk_token" in tokenizer_config: - for key in tokenizer["added_tokens"]: + for key in tokenizer_json["added_tokens"]: if key["content"] == tokenizer_config["unk_token"]: gguf_writer.add_unk_token_id(key["id"]) if "sep_token" in tokenizer_config: - for key in tokenizer["added_tokens"]: + for key in tokenizer_json["added_tokens"]: if key["content"] == tokenizer_config["sep_token"]: gguf_writer.add_sep_token_id(key["id"]) if "pad_token" in tokenizer_config: - for key in tokenizer["added_tokens"]: + for key in tokenizer_json["added_tokens"]: if key["content"] == tokenizer_config["pad_token"]: gguf_writer.add_pad_token_id(key["id"]) @@ -165,11 +216,9 @@ for name in list_vars.keys(): data = list_vars[name].squeeze().numpy() -# print("Process tensor: " + name + " with shape: ", data.shape) # we don't need these if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): -# print(" Skip tensor: " + name) continue n_dims = len(data.shape) @@ -178,16 +227,13 @@ ftype_cur = 0 if ftype != 0: if name.endswith(".weight") and n_dims == 2: -# print(" Converting to float16") data = data.astype(np.float16) ftype_cur = 1 else: -# print(" Converting to float32") data = data.astype(np.float32) ftype_cur = 0 else: if data.dtype != np.float32: -# print(" Converting to float32") data = data.astype(np.float32) ftype_cur = 0 From e6f19ba2404b21e84e1f9dbc5d1b7002a67694e6 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 4 Aug 2023 03:56:37 +0200 Subject: [PATCH 073/242] gptneox-main.cpp : gpt2 bpe tokenizer --- gptneox-main.cpp | 372 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 329 insertions(+), 43 deletions(-) diff --git a/gptneox-main.cpp b/gptneox-main.cpp index 235fca7ce7094..0161d9c2ee7e8 100644 --- a/gptneox-main.cpp +++ b/gptneox-main.cpp @@ -1,6 +1,5 @@ #include "ggml.h" - -#include "gptneox-common.h" +#include "cmpnct_gpt2bpe.hpp" #include #include @@ -11,6 +10,8 @@ #include #include #include +#include +#include #if defined(_MSC_VER) #pragma warning(disable: 4244 4267) // possible loss of data @@ -20,11 +21,11 @@ struct gpt_neox_hparams { size_t n_merges = 0; size_t n_vocab = 0; - int32_t n_ctx = 0; - int32_t n_embd = 0; - int32_t n_head = 0; - int32_t n_layer = 0; - int32_t n_rot = 0; // rotary_pct * (n_embd / n_head) + uint32_t n_ctx = 0; + uint32_t n_embd = 0; + uint32_t n_head = 0; + uint32_t n_layer = 0; + uint32_t n_rot = 0; // rotary_pct * (n_embd / n_head) bool par_res = true; float norm_eps = 1e-5; }; @@ -78,6 +79,241 @@ struct gpt_neox_model { std::map tensors; }; +struct gpt_params { + int32_t seed = -1; // RNG seed + int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); + uint32_t n_predict = 200; // new tokens to predict + uint32_t n_batch = 512; // batch size for prompt processing + + // sampling parameters + int32_t top_k = 40; + float top_p = 1.0f; + float temp = 0.8f; + int32_t repeat_last_n = 64; + float repeat_penalty = 1.02f; + + std::string model = ""; // model path + std::string prompt = ""; + + std::string token_test = ""; + bool interactive = false; + int32_t interactive_port = -1; + int32_t n_gpu_layers = 0; +}; + +void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { + fprintf(stderr, "usage: %s [options]\n", argv[0]); + fprintf(stderr, "\n"); + fprintf(stderr, "options:\n"); + fprintf(stderr, " -h, --help show this help message and exit\n"); + fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n"); + fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); + fprintf(stderr, " -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers); + fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); + fprintf(stderr, " prompt to start generation with (default: random)\n"); + fprintf(stderr, " -f FNAME, --file FNAME\n"); + fprintf(stderr, " load prompt from a file\n"); + fprintf(stderr, " -tt TOKEN_TEST, --token_test TOKEN_TEST\n"); + fprintf(stderr, " test tokenization\n"); + fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d)\n", params.n_predict); + fprintf(stderr, " --top_k N top-k sampling, 0 = n_vocab (default: %d)\n", params.top_k); + fprintf(stderr, " --top_p N top-p sampling (default: %.1f)\n", params.top_p); + fprintf(stderr, " --temp N temperature (default: %.1f)\n", params.temp); + fprintf(stderr, " --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled)\n", params.repeat_last_n); + fprintf(stderr, " --repeat-penalty N penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\n", (double)params.repeat_penalty); + fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch); + fprintf(stderr, " -m FNAME, --model FNAME\n"); + fprintf(stderr, " model path (default: %s)\n", params.model.c_str()); + fprintf(stderr, "\n"); +} + +// Function to check if the next argument exists +std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) { + if (i + 1 < argc && argv[i + 1][0] != '-') { + return argv[++i]; + } else { + fprintf(stderr, "error: %s requires one argument.\n", flag.c_str()); + gpt_print_usage(argc, argv, params); + exit(0); + } +} + +bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { + for (int i = 1; i < argc; i++) { + std::string arg = argv[i]; + + if (arg == "-s" || arg == "--seed") { + params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-t" || arg == "--threads") { + params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") { + params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-p" || arg == "--prompt") { + params.prompt = get_next_arg(i, argc, argv, arg, params); + } else if (arg == "-n" || arg == "--n_predict") { + params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--top_k") { + params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--top_p") { + params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--temp") { + params.temp = std::stof(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--repeat-last-n") { + params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--repeat-penalty") { + params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-b" || arg == "--batch_size") { + params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-m" || arg == "--model") { + params.model = get_next_arg(i, argc, argv, arg, params); + } else if (arg == "-i" || arg == "--interactive") { + params.interactive = true; + } else if (arg == "-ip" || arg == "--interactive-port") { + params.interactive = true; + params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-h" || arg == "--help") { + gpt_print_usage(argc, argv, params); + exit(0); + } else if (arg == "-f" || arg == "--file") { + get_next_arg(i, argc, argv, arg, params); + std::ifstream file(argv[i]); + if (!file) { + fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); + break; + } + std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.prompt)); + if (params.prompt.back() == '\n') { + params.prompt.pop_back(); + } + } else if (arg == "-tt" || arg == "--token_test") { + params.token_test = get_next_arg(i, argc, argv, arg, params); + } + else { + fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); + gpt_print_usage(argc, argv, params); + exit(0); + } + } + + return true; +} + +gpt2bpe_vocab::id sample_top_k_top_p_repeat( + const gpt2bpe_vocab & vocab, + const float * logits, + const int32_t * last_n_tokens_data, + size_t last_n_tokens_data_size, + int top_k, + double top_p, + double temp, + int repeat_last_n, + float repeat_penalty, + std::mt19937 & rng) { + + int n_logits = vocab.id_to_token.size(); + + const auto * plogits = logits; + + const auto last_n_tokens = std::vector(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size); + + if (temp <= 0) { + // select the token with the highest logit directly + float max_logit = plogits[0]; + gpt2bpe_vocab::id max_id = 0; + + for (int i = 1; i < n_logits; ++i) { + if (plogits[i] > max_logit) { + max_logit = plogits[i]; + max_id = i; + } + } + return max_id; + } + + + std::vector> logits_id; + logits_id.reserve(n_logits); + + { + const float scale = 1.0f/temp; + for (int i = 0; i < n_logits; ++i) { + // repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858) + // credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main + if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) { + // if score < 0 then repetition penalty has to multiplied to reduce the previous token probability + if (plogits[i] < 0.0f) { + logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i)); + } else { + logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i)); + } + } else { + logits_id.push_back(std::make_pair(plogits[i]*scale, i)); + } + } + } + + // find the top K tokens + std::partial_sort( + logits_id.begin(), + logits_id.begin() + top_k, logits_id.end(), + [](const std::pair & a, const std::pair & b) { + return a.first > b.first; + }); + + logits_id.resize(top_k); + + double maxl = -INFINITY; + for (const auto & kv : logits_id) { + maxl = std::max(maxl, kv.first); + } + + // compute probs for the top K tokens + std::vector probs; + probs.reserve(logits_id.size()); + + double sum = 0.0; + for (const auto & kv : logits_id) { + double p = exp(kv.first - maxl); + probs.push_back(p); + sum += p; + } + + // normalize the probs + for (auto & p : probs) { + p /= sum; + } + + if (top_p < 1.0f) { + double cumsum = 0.0f; + for (int i = 0; i < top_k; i++) { + cumsum += probs[i]; + if (cumsum >= top_p) { + top_k = i + 1; + probs.resize(top_k); + logits_id.resize(top_k); + break; + } + } + + cumsum = 1.0/cumsum; + for (int i = 0; i < (int) probs.size(); i++) { + probs[i] *= cumsum; + } + } + +// printf("\n"); +// for (int i = 0; i < (int) probs.size(); i++) { +// for (int i = 0; i < 10; i++) { +// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]); +// } + + std::discrete_distribution<> dist(probs.begin(), probs.end()); + int idx = dist(rng); + + return logits_id[idx].second; + +} + struct ggml_tensor * get_tensor_ex( struct ggml_context * ctx, std::string name){ struct ggml_tensor * cur = ggml_get_tensor(ctx, name.c_str()); @@ -91,7 +327,7 @@ struct ggml_tensor * get_tensor_ex( struct ggml_context * ctx, std::string name) } // load the model's weights from a file -bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_vocab & vocab) { +bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2bpe_vocab & vocab) { printf("%s: loading model from '%s'..\n", __func__, fname.c_str()); model.ctx = NULL; @@ -115,7 +351,7 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_ fprintf(stdout, "%s: gguf data offset = %zu\n", __func__, gguf_get_data_offset(ggufctx)); // print all kv - if( false ) + #if 0 { const int n_kv = gguf_get_n_kv(ggufctx); @@ -127,6 +363,7 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_ fprintf(stdout, "%s: kv[%d]: key = %s\n", __func__, i, key); } } + #endif // print some standard metadata { @@ -249,20 +486,47 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_ // TEMP until a better bpe tokenizer is implemented - word = replace(word, "Ġ", " "); - word = replace(word, "Ċ", "\n"); +// word = replace(word, "Ġ", " "); +// word = replace(word, "Ċ", "\n"); +// printf("token %d = '%s'\n",i,word.c_str() ); vocab.token_to_id[word] = i; vocab.id_to_token[i] = word; + } - keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.bos_token_id"); if( keyidx != -1 ) { printf("bos id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); } - keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.eos_token_id"); if( keyidx != -1 ) { printf("eos id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); } - keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.unknown_token_id"); if( keyidx != -1 ) { printf("unk id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); } - keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.separator_token_id"); if( keyidx != -1 ) { printf("sep id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); } - keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.padding_token_id"); if( keyidx != -1 ) { printf("pad id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); } + std::vector> bpe_merges; + + for (size_t i = 0; i < hparams.n_merges; i++) { + std::string word = gguf_get_arr_str(ggufctx, merges_keyidx, i); + + // Split the merges + std::string first, second; + size_t pos = word.find(' ', 1); // Start the search from the second character + if (pos != std::string::npos) { + first = word.substr(0, pos); + second = word.substr(pos + 1); + } + + bpe_merges.push_back(std::make_pair(first, second)); + } + + vocab.populate_bpe_ranks(bpe_merges); + + + keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.bos_token_id"); if( keyidx != -1 ) { vocab.special_bos_id = (int32_t)gguf_get_val_u32(ggufctx, keyidx); vocab.special_have_bos=true; } + keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.eos_token_id"); if( keyidx != -1 ) { vocab.special_eos_id = (int32_t)gguf_get_val_u32(ggufctx, keyidx); vocab.special_have_eos=true; } + keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.unknown_token_id"); if( keyidx != -1 ) { vocab.special_unk_id = (int32_t)gguf_get_val_u32(ggufctx, keyidx); vocab.special_have_unk=true; } + keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.separator_token_id"); if( keyidx != -1 ) { vocab.special_sep_id = (int32_t)gguf_get_val_u32(ggufctx, keyidx); vocab.special_have_sep=true; } + keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.padding_token_id"); if( keyidx != -1 ) { vocab.special_pad_id = (int32_t)gguf_get_val_u32(ggufctx, keyidx); vocab.special_have_pad=true; } + + if( vocab.special_have_bos ) { fprintf(stdout, "%s: bos token = %d '%s'\n", __func__, vocab.special_bos_id, vocab.id_to_token[vocab.special_bos_id].c_str() ); } + if( vocab.special_have_eos ) { fprintf(stdout, "%s: eos token = %d '%s'\n", __func__, vocab.special_eos_id, vocab.id_to_token[vocab.special_eos_id].c_str() ); } + if( vocab.special_have_unk ) { fprintf(stdout, "%s: unk token = %d '%s'\n", __func__, vocab.special_unk_id, vocab.id_to_token[vocab.special_unk_id].c_str() ); } + if( vocab.special_have_sep ) { fprintf(stdout, "%s: sep token = %d '%s'\n", __func__, vocab.special_sep_id, vocab.id_to_token[vocab.special_sep_id].c_str() ); } + if( vocab.special_have_pad ) { fprintf(stdout, "%s: pad token = %d '%s'\n", __func__, vocab.special_pad_id, vocab.id_to_token[vocab.special_pad_id].c_str() ); } } @@ -272,7 +536,7 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_ printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0)); // print tensor info - if( false ) + #if 0 { const int n_tensors = gguf_get_n_tensors(ggufctx); @@ -285,7 +549,7 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_ fprintf(stdout, "%s: tensor[%d]: name = %s, offset = %zu\n", __func__, i, name, offset); } } - + #endif // prepare memory for the weights { @@ -435,7 +699,7 @@ bool gpt_neox_eval( const gpt_neox_model & model, const int n_threads, const int n_past, - const std::vector & embd_inp, + const std::vector & embd_inp, std::vector & embd_w, size_t & mem_per_token) { const int N = embd_inp.size(); @@ -687,20 +951,9 @@ int main(int argc, char ** argv) { return 1; } - if (params.seed < 0) { - params.seed = time(NULL); - } - - printf("%s: seed = %d\n", __func__, params.seed); - - std::mt19937 rng(params.seed); - if (params.prompt.empty()) { - params.prompt = gpt_random_prompt(rng); - } - int64_t t_load_us = 0; - gpt_vocab vocab; + gpt2bpe_vocab vocab; gpt_neox_model model; // load the model @@ -716,8 +969,29 @@ int main(int argc, char ** argv) { } - uint32_t eos_token_id = 0; - int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.eos_token_id"); if( keyidx != -1 ) { eos_token_id = gguf_get_val_u32(ggufctx, keyidx); } + if (params.seed < 0) { + params.seed = time(NULL); + } + + if (params.top_k == 0) { + params.top_k = model.hparams.n_vocab; + } + + printf("%s: seed = %d\n", __func__, params.seed); + printf("%s: temp = %.3f\n", __func__, params.temp); + printf("%s: top_k = %d\n", __func__, params.top_k); + printf("%s: top_p = %.3f\n", __func__, params.top_p); + printf("%s: repeat_last_n = %d\n", __func__, params.repeat_last_n); + printf("%s: repeat_penalty = %.3f\n", __func__, params.repeat_penalty); + + std::mt19937 rng(params.seed); + + if (params.prompt.empty()) { + params.prompt = "Once upon"; + } + + std::vector last_n_tokens(model.hparams.n_ctx); + std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0); int n_past = 0; @@ -727,23 +1001,29 @@ int main(int argc, char ** argv) { std::vector logits; // tokenize the prompt - std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt); + std::vector embd_inp = gpt2bpe_tokenize(vocab, params.prompt,false, false); params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size()); printf("%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size()); - for (int i = 0; i < embd_inp.size(); i++) { - printf("%s: token[%d] = %6d, %s\n", __func__, i, embd_inp[i], vocab.id_to_token.at(embd_inp[i]).c_str()); +// for (size_t i = 0; i < embd_inp.size(); i++) { +// printf("%s: token[%zu] = %6d, %s\n", __func__, i, embd_inp[i], vocab.id_to_token[embd_inp[i]].c_str()); +// } + + if( model.hparams.n_ctx < params.n_predict+embd_inp.size() ) { + params.n_predict = model.hparams.n_ctx-embd_inp.size(); } + + printf("%s: n_predict = %d\n", __func__, params.n_predict); printf("\n"); - std::vector embd; + std::vector embd; // determine the required inference memory per token: size_t mem_per_token = 0; gpt_neox_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token); - for (int i = embd.size(); i < embd_inp.size() + params.n_predict; i++) { + for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) { // predict if (embd.size() > 0) { const int64_t t_start_us = ggml_time_us(); @@ -764,15 +1044,21 @@ int main(int argc, char ** argv) { const int top_k = params.top_k; const float top_p = params.top_p; const float temp = params.temp; + const int repeat_last_n = params.repeat_last_n; + const float repeat_penalty = params.repeat_penalty; const int n_vocab = model.hparams.n_vocab; - gpt_vocab::id id = 0; + gpt2bpe_vocab::id id = 0; { const int64_t t_start_sample_us = ggml_time_us(); - id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng); +// id = sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, repeat_last_n, repeat_penalty, rng); + id = sample_top_k_top_p_repeat(vocab, logits.data() + (logits.size() - n_vocab), last_n_tokens.data(), last_n_tokens.size(), top_k, top_p, temp, repeat_last_n, repeat_penalty, rng); + + last_n_tokens.erase(last_n_tokens.begin()); + last_n_tokens.push_back(id); t_sample_us += ggml_time_us() - t_start_sample_us; } @@ -781,7 +1067,7 @@ int main(int argc, char ** argv) { embd.push_back(id); } else { // if here, it means we are still processing the input prompt - for (int k = i; k < embd_inp.size(); k++) { + for (size_t k = i; k < embd_inp.size(); k++) { embd.push_back(embd_inp[k]); if (embd.size() > params.n_batch) { break; @@ -792,12 +1078,12 @@ int main(int argc, char ** argv) { // display text for (auto id : embd) { - printf("%s", vocab.id_to_token[id].c_str()); + printf("%s", vocab.id_to_token[id].c_str() ); } fflush(stdout); // end of text token - if (embd.back() == eos_token_id) { + if (vocab.special_have_eos && embd.back() == vocab.special_eos_id) { break; } } From 5d98989cf647b39b993cfd6dcf271cc282eea58f Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 4 Aug 2023 03:58:44 +0200 Subject: [PATCH 074/242] gpt2 bpe tokenizer (handles merges and unicode) --- cmpnct_gpt2bpe.hpp | 1011 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1011 insertions(+) create mode 100644 cmpnct_gpt2bpe.hpp diff --git a/cmpnct_gpt2bpe.hpp b/cmpnct_gpt2bpe.hpp new file mode 100644 index 0000000000000..0743a294ac0ff --- /dev/null +++ b/cmpnct_gpt2bpe.hpp @@ -0,0 +1,1011 @@ +#ifndef CMPNCT_GPT2BPE +#define CMPNCT_GPT2BPE + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/** + * https://github.com/cmp-nct/ggllm.cpp + * Minimal library for high performance handling and categorization of UTF8 strings and characters + * Using std::string + */ + +enum CNCTCharType { + DIGIT, // a numerical char in any language + LETTER, // a letter in any language + WHITESPACE, // any form of whitespace + ACCENT_MARK, // letter modifiers like ´ in é + PUNCTUATION, // punctuation including brackets + SYMBOL, // math, currency, other symbols + CONTROL, // control characters + MIXED, // a mix of the above + UNIDENTIFIED // something more exotic like emoji or separators +}; + +struct CNCTUnicode; + +struct CNCTString { + std::string str; + size_t utf8_chars; + + CNCTCharType char_type=UNIDENTIFIED; + bool is_sequential=false; + + size_t seq_offset_bytes=0; + size_t seq_offset_utf8_chars=0; + + bool operator==(const std::string &other) const; + bool operator==(const char other) const; + bool operator==(const CNCTString &other) const; + CNCTString &operator+=(const std::string &other); + CNCTString &operator+=(const char other); + friend CNCTString operator+(CNCTString lhs, const std::string &rhs); + friend CNCTString operator+(CNCTString lhs, const char rhs); + CNCTString& operator+=(const CNCTString& other); + friend CNCTString operator+(CNCTString lhs, const CNCTString& rhs); +}; + +struct CNCTUnicode { + static bool check_code_range(int c, const std::vector>& ranges); + static CNCTCharType get_code_type(int c); + static CNCTCharType get_code_type(const std::string &utf8_char); + static int utf8_len(const char c); + static int strlen_utf8(std::string src); + static std::vector split_utf8(const std::string &src); + static std::vector split_utf8_enhanced(const std::string &src); + static CNCTCharType string_identify(const std::string& str); + static bool string_test(const std::string& str, CNCTCharType chartype); +}; + +static const std::vector> digit_ranges = { +{0x30, 0x39}, {0xB2, 0xB3}, {0xB9, 0xB9}, {0x660, 0x669}, {0x6F0, 0x6F9}, {0x7C0, 0x7C9}, {0x966, 0x96F}, {0x9E6, 0x9EF}, {0xA66, 0xA6F}, {0xAE6, 0xAEF}, {0xB66, 0xB6F}, {0xBE6, 0xBEF}, {0xC66, 0xC6F}, {0xCE6, 0xCEF}, {0xD66, 0xD6F}, {0xDE6, 0xDEF}, {0xE50, 0xE59}, {0xED0, 0xED9}, {0xF20, 0xF29}, {0x1040, 0x1049}, {0x1090, 0x1099}, {0x1369, 0x1371}, {0x17E0, 0x17E9}, {0x1810, 0x1819}, {0x1946, 0x194F}, {0x19D0, 0x19DA}, {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, {0x1B50, 0x1B59}, {0x1BB0, 0x1BB9}, {0x1C40, 0x1C49}, {0x1C50, 0x1C59}, {0x2070, 0x2070}, {0x2074, 0x2079}, {0x2080, 0x2089}, {0x2460, 0x2468}, {0x2474, 0x247C}, {0x2488, 0x2490}, {0x24EA, 0x24EA}, {0x24F5, 0x24FD}, {0x24FF, 0x24FF}, {0x2776, 0x277E}, {0x2780, 0x2788}, {0x278A, 0x2792}, {0xA620, 0xA629}, {0xA8D0, 0xA8D9}, {0xA900, 0xA909}, {0xA9D0, 0xA9D9}, {0xA9F0, 0xA9F9}, {0xAA50, 0xAA59}, {0xABF0, 0xABF9}, {0xFF10, 0xFF19}, {0x104A0, 0x104A9}, {0x10A40, 0x10A43}, {0x10D30, 0x10D39}, {0x10E60, 0x10E68}, {0x11052, 0x1105A}, {0x11066, 0x1106F}, {0x110F0, 0x110F9}, {0x11136, 0x1113F}, {0x111D0, 0x111D9}, {0x112F0, 0x112F9}, {0x11450, 0x11459}, {0x114D0, 0x114D9}, {0x11650, 0x11659}, {0x116C0, 0x116C9}, {0x11730, 0x11739}, {0x118E0, 0x118E9}, {0x11950, 0x11959}, {0x11C50, 0x11C59}, {0x11D50, 0x11D59}, {0x11DA0, 0x11DA9}, {0x16A60, 0x16A69}, {0x16B50, 0x16B59}, {0x1D7CE, 0x1D7FF}, {0x1E140, 0x1E149}, {0x1E2F0, 0x1E2F9}, {0x1E950, 0x1E959}, {0x1F100, 0x1F10A}, {0x1FBF0, 0x1FBF9}, }; + +static const std::vector> letter_ranges = { +{0x41, 0x5A}, {0x61, 0x7A}, {0xAA, 0xAA}, {0xB5, 0xB5}, {0xBA, 0xBA}, {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, {0x370, 0x374}, {0x376, 0x377}, {0x37A, 0x37D}, {0x37F, 0x37F}, {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, {0x559, 0x559}, {0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, {0x620, 0x64A}, {0x66E, 0x66F}, {0x671, 0x6D3}, {0x6D5, 0x6D5}, {0x6E5, 0x6E6}, {0x6EE, 0x6EF}, {0x6FA, 0x6FC}, {0x6FF, 0x6FF}, {0x710, 0x710}, {0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, {0x7CA, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, {0x81A, 0x81A}, {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, {0x860, 0x86A}, {0x8A0, 0x8B4}, {0x8B6, 0x8C7}, {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, {0x958, 0x961}, {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, {0x993, 0x9A8}, {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, {0x9B6, 0x9B9}, {0x9BD, 0x9BD}, {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, {0x9F0, 0x9F1}, {0x9FC, 0x9FC}, {0xA05, 0xA0A}, {0xA0F, 0xA10}, {0xA13, 0xA28}, {0xA2A, 0xA30}, {0xA32, 0xA33}, {0xA35, 0xA36}, {0xA38, 0xA39}, {0xA59, 0xA5C}, {0xA5E, 0xA5E}, {0xA72, 0xA74}, {0xA85, 0xA8D}, {0xA8F, 0xA91}, {0xA93, 0xAA8}, {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, {0xAB5, 0xAB9}, {0xABD, 0xABD}, {0xAD0, 0xAD0}, {0xAE0, 0xAE1}, {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, {0xB13, 0xB28}, {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, {0xB5C, 0xB5D}, {0xB5F, 0xB61}, {0xB71, 0xB71}, {0xB83, 0xB83}, {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, {0xB99, 0xB9A}, {0xB9C, 0xB9C}, {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, {0xBA8, 0xBAA}, {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, {0xC05, 0xC0C}, {0xC0E, 0xC10}, {0xC12, 0xC28}, {0xC2A, 0xC39}, {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC60, 0xC61}, {0xC80, 0xC80}, {0xC85, 0xC8C}, {0xC8E, 0xC90}, {0xC92, 0xCA8}, {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, {0xCBD, 0xCBD}, {0xCDE, 0xCDE}, {0xCE0, 0xCE1}, {0xCF1, 0xCF2}, {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, {0xD3D, 0xD3D}, {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD5F, 0xD61}, {0xD7A, 0xD7F}, {0xD85, 0xD96}, {0xD9A, 0xDB1}, {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, {0xDC0, 0xDC6}, {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A}, {0xE8C, 0xEA3}, {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, {0xEC0, 0xEC4}, {0xEC6, 0xEC6}, {0xEDC, 0xEDF}, {0xF00, 0xF00}, {0xF40, 0xF47}, {0xF49, 0xF6C}, {0xF88, 0xF8C}, {0x1000, 0x102A}, {0x103F, 0x103F}, {0x1050, 0x1055}, {0x105A, 0x105D}, {0x1061, 0x1061}, {0x1065, 0x1066}, {0x106E, 0x1070}, {0x1075, 0x1081}, {0x108E, 0x108E}, {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, {0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FC, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258}, {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, {0x1380, 0x138F}, {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C}, {0x166F, 0x167F}, {0x1681, 0x169A}, {0x16A0, 0x16EA}, {0x16F1, 0x16F8}, {0x1700, 0x170C}, {0x170E, 0x1711}, {0x1720, 0x1731}, {0x1740, 0x1751}, {0x1760, 0x176C}, {0x176E, 0x1770}, {0x1780, 0x17B3}, {0x17D7, 0x17D7}, {0x17DC, 0x17DC}, {0x1820, 0x1878}, {0x1880, 0x1884}, {0x1887, 0x18A8}, {0x18AA, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1950, 0x196D}, {0x1970, 0x1974}, {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x1A00, 0x1A16}, {0x1A20, 0x1A54}, {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, {0x1B45, 0x1B4B}, {0x1B83, 0x1BA0}, {0x1BAE, 0x1BAF}, {0x1BBA, 0x1BE5}, {0x1C00, 0x1C23}, {0x1C4D, 0x1C4F}, {0x1C5A, 0x1C7D}, {0x1C80, 0x1C88}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, {0x1D00, 0x1DBF}, {0x1E00, 0x1F15}, {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, {0x1FB6, 0x1FBC}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x2071, 0x2071}, {0x207F, 0x207F}, {0x2090, 0x209C}, {0x2102, 0x2102}, {0x2107, 0x2107}, {0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, {0x2124, 0x2124}, {0x2126, 0x2126}, {0x2128, 0x2128}, {0x212A, 0x212D}, {0x212F, 0x2139}, {0x213C, 0x213F}, {0x2145, 0x2149}, {0x214E, 0x214E}, {0x2183, 0x2184}, {0x2C00, 0x2C2E}, {0x2C30, 0x2C5E}, {0x2C60, 0x2CE4}, {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, {0x2D00, 0x2D25}, {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F}, {0x2D80, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3006}, {0x3031, 0x3035}, {0x303B, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3400, 0x4DBF}, {0x4E00, 0x9FFC}, {0xA000, 0xA48C}, {0xA4D0, 0xA4FD}, {0xA500, 0xA60C}, {0xA610, 0xA61F}, {0xA62A, 0xA62B}, {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6E5}, {0xA717, 0xA71F}, {0xA722, 0xA788}, {0xA78B, 0xA7BF}, {0xA7C2, 0xA7CA}, {0xA7F5, 0xA801}, {0xA803, 0xA805}, {0xA807, 0xA80A}, {0xA80C, 0xA822}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, {0xA8F2, 0xA8F7}, {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE}, {0xA90A, 0xA925}, {0xA930, 0xA946}, {0xA960, 0xA97C}, {0xA984, 0xA9B2}, {0xA9CF, 0xA9CF}, {0xA9E0, 0xA9E4}, {0xA9E6, 0xA9EF}, {0xA9FA, 0xA9FE}, {0xAA00, 0xAA28}, {0xAA40, 0xAA42}, {0xAA44, 0xAA4B}, {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4}, {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, {0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x10300, 0x1031F}, {0x1032D, 0x10340}, {0x10342, 0x10349}, {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, {0x103C8, 0x103CF}, {0x10400, 0x1049D}, {0x104B0, 0x104D3}, {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, {0x10600, 0x10736}, {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C}, {0x1083F, 0x10855}, {0x10860, 0x10876}, {0x10880, 0x1089E}, {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x10900, 0x10915}, {0x10920, 0x10939}, {0x10980, 0x109B7}, {0x109BE, 0x109BF}, {0x10A00, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, {0x10A60, 0x10A7C}, {0x10A80, 0x10A9C}, {0x10AC0, 0x10AC7}, {0x10AC9, 0x10AE4}, {0x10B00, 0x10B35}, {0x10B40, 0x10B55}, {0x10B60, 0x10B72}, {0x10B80, 0x10B91}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, {0x10D00, 0x10D23}, {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, {0x10F00, 0x10F1C}, {0x10F27, 0x10F27}, {0x10F30, 0x10F45}, {0x10FB0, 0x10FC4}, {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x11103, 0x11126}, {0x11144, 0x11144}, {0x11147, 0x11147}, {0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, {0x111C1, 0x111C4}, {0x111DA, 0x111DA}, {0x111DC, 0x111DC}, {0x11200, 0x11211}, {0x11213, 0x1122B}, {0x11280, 0x11286}, {0x11288, 0x11288}, {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339}, {0x1133D, 0x1133D}, {0x11350, 0x11350}, {0x1135D, 0x11361}, {0x11400, 0x11434}, {0x11447, 0x1144A}, {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, {0x11580, 0x115AE}, {0x115D8, 0x115DB}, {0x11600, 0x1162F}, {0x11644, 0x11644}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, {0x11700, 0x1171A}, {0x11800, 0x1182B}, {0x118A0, 0x118DF}, {0x118FF, 0x11906}, {0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x1192F}, {0x1193F, 0x1193F}, {0x11941, 0x11941}, {0x119A0, 0x119A7}, {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, {0x11A5C, 0x11A89}, {0x11A9D, 0x11A9D}, {0x11AC0, 0x11AF8}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, {0x11C40, 0x11C40}, {0x11C72, 0x11C8F}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11EE0, 0x11EF2}, {0x11FB0, 0x11FB0}, {0x12000, 0x12399}, {0x12480, 0x12543}, {0x13000, 0x1342E}, {0x14400, 0x14646}, {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16E40, 0x16E7F}, {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, {0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08}, {0x1B000, 0x1B11E}, {0x1B150, 0x1B152}, {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, {0x1E14E, 0x1E14E}, {0x1E2C0, 0x1E2EB}, {0x1E800, 0x1E8C4}, {0x1E900, 0x1E943}, {0x1E94B, 0x1E94B}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x20000, 0x2A6DD}, {0x2A700, 0x2B734}, {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, {0x2CEB0, 0x2EBE0}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, }; + +static const std::vector> whitespace_ranges = { +{0x9, 0xD}, {0x1C, 0x20}, {0x85, 0x85}, {0xA0, 0xA0}, {0x1680, 0x1680}, {0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F}, {0x205F, 0x205F}, {0x3000, 0x3000}, }; + +static const std::vector> accent_mark_ranges = { +{0x300, 0x36F}, {0x483, 0x489}, {0x591, 0x5BD}, {0x5BF, 0x5BF}, {0x5C1, 0x5C2}, {0x5C4, 0x5C5}, {0x5C7, 0x5C7}, {0x610, 0x61A}, {0x64B, 0x65F}, {0x670, 0x670}, {0x6D6, 0x6DC}, {0x6DF, 0x6E4}, {0x6E7, 0x6E8}, {0x6EA, 0x6ED}, {0x711, 0x711}, {0x730, 0x74A}, {0x7A6, 0x7B0}, {0x7EB, 0x7F3}, {0x7FD, 0x7FD}, {0x816, 0x819}, {0x81B, 0x823}, {0x825, 0x827}, {0x829, 0x82D}, {0x859, 0x85B}, {0x8D3, 0x8E1}, {0x8E3, 0x903}, {0x93A, 0x93C}, {0x93E, 0x94F}, {0x951, 0x957}, {0x962, 0x963}, {0x981, 0x983}, {0x9BC, 0x9BC}, {0x9BE, 0x9C4}, {0x9C7, 0x9C8}, {0x9CB, 0x9CD}, {0x9D7, 0x9D7}, {0x9E2, 0x9E3}, {0x9FE, 0x9FE}, {0xA01, 0xA03}, {0xA3C, 0xA3C}, {0xA3E, 0xA42}, {0xA47, 0xA48}, {0xA4B, 0xA4D}, {0xA51, 0xA51}, {0xA70, 0xA71}, {0xA75, 0xA75}, {0xA81, 0xA83}, {0xABC, 0xABC}, {0xABE, 0xAC5}, {0xAC7, 0xAC9}, {0xACB, 0xACD}, {0xAE2, 0xAE3}, {0xAFA, 0xAFF}, {0xB01, 0xB03}, {0xB3C, 0xB3C}, {0xB3E, 0xB44}, {0xB47, 0xB48}, {0xB4B, 0xB4D}, {0xB55, 0xB57}, {0xB62, 0xB63}, {0xB82, 0xB82}, {0xBBE, 0xBC2}, {0xBC6, 0xBC8}, {0xBCA, 0xBCD}, {0xBD7, 0xBD7}, {0xC00, 0xC04}, {0xC3E, 0xC44}, {0xC46, 0xC48}, {0xC4A, 0xC4D}, {0xC55, 0xC56}, {0xC62, 0xC63}, {0xC81, 0xC83}, {0xCBC, 0xCBC}, {0xCBE, 0xCC4}, {0xCC6, 0xCC8}, {0xCCA, 0xCCD}, {0xCD5, 0xCD6}, {0xCE2, 0xCE3}, {0xD00, 0xD03}, {0xD3B, 0xD3C}, {0xD3E, 0xD44}, {0xD46, 0xD48}, {0xD4A, 0xD4D}, {0xD57, 0xD57}, {0xD62, 0xD63}, {0xD81, 0xD83}, {0xDCA, 0xDCA}, {0xDCF, 0xDD4}, {0xDD6, 0xDD6}, {0xDD8, 0xDDF}, {0xDF2, 0xDF3}, {0xE31, 0xE31}, {0xE34, 0xE3A}, {0xE47, 0xE4E}, {0xEB1, 0xEB1}, {0xEB4, 0xEBC}, {0xEC8, 0xECD}, {0xF18, 0xF19}, {0xF35, 0xF35}, {0xF37, 0xF37}, {0xF39, 0xF39}, {0xF3E, 0xF3F}, {0xF71, 0xF84}, {0xF86, 0xF87}, {0xF8D, 0xF97}, {0xF99, 0xFBC}, {0xFC6, 0xFC6}, {0x102B, 0x103E}, {0x1056, 0x1059}, {0x105E, 0x1060}, {0x1062, 0x1064}, {0x1067, 0x106D}, {0x1071, 0x1074}, {0x1082, 0x108D}, {0x108F, 0x108F}, {0x109A, 0x109D}, {0x135D, 0x135F}, {0x1712, 0x1714}, {0x1732, 0x1734}, {0x1752, 0x1753}, {0x1772, 0x1773}, {0x17B4, 0x17D3}, {0x17DD, 0x17DD}, {0x180B, 0x180D}, {0x1885, 0x1886}, {0x18A9, 0x18A9}, {0x1920, 0x192B}, {0x1930, 0x193B}, {0x1A17, 0x1A1B}, {0x1A55, 0x1A5E}, {0x1A60, 0x1A7C}, {0x1A7F, 0x1A7F}, {0x1AB0, 0x1AC0}, {0x1B00, 0x1B04}, {0x1B34, 0x1B44}, {0x1B6B, 0x1B73}, {0x1B80, 0x1B82}, {0x1BA1, 0x1BAD}, {0x1BE6, 0x1BF3}, {0x1C24, 0x1C37}, {0x1CD0, 0x1CD2}, {0x1CD4, 0x1CE8}, {0x1CED, 0x1CED}, {0x1CF4, 0x1CF4}, {0x1CF7, 0x1CF9}, {0x1DC0, 0x1DF9}, {0x1DFB, 0x1DFF}, {0x20D0, 0x20F0}, {0x2CEF, 0x2CF1}, {0x2D7F, 0x2D7F}, {0x2DE0, 0x2DFF}, {0x302A, 0x302F}, {0x3099, 0x309A}, {0xA66F, 0xA672}, {0xA674, 0xA67D}, {0xA69E, 0xA69F}, {0xA6F0, 0xA6F1}, {0xA802, 0xA802}, {0xA806, 0xA806}, {0xA80B, 0xA80B}, {0xA823, 0xA827}, {0xA82C, 0xA82C}, {0xA880, 0xA881}, {0xA8B4, 0xA8C5}, {0xA8E0, 0xA8F1}, {0xA8FF, 0xA8FF}, {0xA926, 0xA92D}, {0xA947, 0xA953}, {0xA980, 0xA983}, {0xA9B3, 0xA9C0}, {0xA9E5, 0xA9E5}, {0xAA29, 0xAA36}, {0xAA43, 0xAA43}, {0xAA4C, 0xAA4D}, {0xAA7B, 0xAA7D}, {0xAAB0, 0xAAB0}, {0xAAB2, 0xAAB4}, {0xAAB7, 0xAAB8}, {0xAABE, 0xAABF}, {0xAAC1, 0xAAC1}, {0xAAEB, 0xAAEF}, {0xAAF5, 0xAAF6}, {0xABE3, 0xABEA}, {0xABEC, 0xABED}, {0xFB1E, 0xFB1E}, {0xFE00, 0xFE0F}, {0xFE20, 0xFE2F}, {0x101FD, 0x101FD}, {0x102E0, 0x102E0}, {0x10376, 0x1037A}, {0x10A01, 0x10A03}, {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F}, {0x10A38, 0x10A3A}, {0x10A3F, 0x10A3F}, {0x10AE5, 0x10AE6}, {0x10D24, 0x10D27}, {0x10EAB, 0x10EAC}, {0x10F46, 0x10F50}, {0x11000, 0x11002}, {0x11038, 0x11046}, {0x1107F, 0x11082}, {0x110B0, 0x110BA}, {0x11100, 0x11102}, {0x11127, 0x11134}, {0x11145, 0x11146}, {0x11173, 0x11173}, {0x11180, 0x11182}, {0x111B3, 0x111C0}, {0x111C9, 0x111CC}, {0x111CE, 0x111CF}, {0x1122C, 0x11237}, {0x1123E, 0x1123E}, {0x112DF, 0x112EA}, {0x11300, 0x11303}, {0x1133B, 0x1133C}, {0x1133E, 0x11344}, {0x11347, 0x11348}, {0x1134B, 0x1134D}, {0x11357, 0x11357}, {0x11362, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374}, {0x11435, 0x11446}, {0x1145E, 0x1145E}, {0x114B0, 0x114C3}, {0x115AF, 0x115B5}, {0x115B8, 0x115C0}, {0x115DC, 0x115DD}, {0x11630, 0x11640}, {0x116AB, 0x116B7}, {0x1171D, 0x1172B}, {0x1182C, 0x1183A}, {0x11930, 0x11935}, {0x11937, 0x11938}, {0x1193B, 0x1193E}, {0x11940, 0x11940}, {0x11942, 0x11943}, {0x119D1, 0x119D7}, {0x119DA, 0x119E0}, {0x119E4, 0x119E4}, {0x11A01, 0x11A0A}, {0x11A33, 0x11A39}, {0x11A3B, 0x11A3E}, {0x11A47, 0x11A47}, {0x11A51, 0x11A5B}, {0x11A8A, 0x11A99}, {0x11C2F, 0x11C36}, {0x11C38, 0x11C3F}, {0x11C92, 0x11CA7}, {0x11CA9, 0x11CB6}, {0x11D31, 0x11D36}, {0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D}, {0x11D3F, 0x11D45}, {0x11D47, 0x11D47}, {0x11D8A, 0x11D8E}, {0x11D90, 0x11D91}, {0x11D93, 0x11D97}, {0x11EF3, 0x11EF6}, {0x16AF0, 0x16AF4}, {0x16B30, 0x16B36}, {0x16F4F, 0x16F4F}, {0x16F51, 0x16F87}, {0x16F8F, 0x16F92}, {0x16FE4, 0x16FE4}, {0x16FF0, 0x16FF1}, {0x1BC9D, 0x1BC9E}, {0x1D165, 0x1D169}, {0x1D16D, 0x1D172}, {0x1D17B, 0x1D182}, {0x1D185, 0x1D18B}, {0x1D1AA, 0x1D1AD}, {0x1D242, 0x1D244}, {0x1DA00, 0x1DA36}, {0x1DA3B, 0x1DA6C}, {0x1DA75, 0x1DA75}, {0x1DA84, 0x1DA84}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006}, {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, {0x1E026, 0x1E02A}, {0x1E130, 0x1E136}, {0x1E2EC, 0x1E2EF}, {0x1E8D0, 0x1E8D6}, {0x1E944, 0x1E94A}, {0xE0100, 0xE01EF}, }; + +static const std::vector> punctuation_ranges = { +{0x21, 0x23}, {0x25, 0x2A}, {0x2C, 0x2F}, {0x3A, 0x3B}, {0x3F, 0x40}, {0x5B, 0x5D}, {0x5F, 0x5F}, {0x7B, 0x7B}, {0x7D, 0x7D}, {0xA1, 0xA1}, {0xA7, 0xA7}, {0xAB, 0xAB}, {0xB6, 0xB7}, {0xBB, 0xBB}, {0xBF, 0xBF}, {0x37E, 0x37E}, {0x387, 0x387}, {0x55A, 0x55F}, {0x589, 0x58A}, {0x5BE, 0x5BE}, {0x5C0, 0x5C0}, {0x5C3, 0x5C3}, {0x5C6, 0x5C6}, {0x5F3, 0x5F4}, {0x609, 0x60A}, {0x60C, 0x60D}, {0x61B, 0x61B}, {0x61E, 0x61F}, {0x66A, 0x66D}, {0x6D4, 0x6D4}, {0x700, 0x70D}, {0x7F7, 0x7F9}, {0x830, 0x83E}, {0x85E, 0x85E}, {0x964, 0x965}, {0x970, 0x970}, {0x9FD, 0x9FD}, {0xA76, 0xA76}, {0xAF0, 0xAF0}, {0xC77, 0xC77}, {0xC84, 0xC84}, {0xDF4, 0xDF4}, {0xE4F, 0xE4F}, {0xE5A, 0xE5B}, {0xF04, 0xF12}, {0xF14, 0xF14}, {0xF3A, 0xF3D}, {0xF85, 0xF85}, {0xFD0, 0xFD4}, {0xFD9, 0xFDA}, {0x104A, 0x104F}, {0x10FB, 0x10FB}, {0x1360, 0x1368}, {0x1400, 0x1400}, {0x166E, 0x166E}, {0x169B, 0x169C}, {0x16EB, 0x16ED}, {0x1735, 0x1736}, {0x17D4, 0x17D6}, {0x17D8, 0x17DA}, {0x1800, 0x180A}, {0x1944, 0x1945}, {0x1A1E, 0x1A1F}, {0x1AA0, 0x1AA6}, {0x1AA8, 0x1AAD}, {0x1B5A, 0x1B60}, {0x1BFC, 0x1BFF}, {0x1C3B, 0x1C3F}, {0x1C7E, 0x1C7F}, {0x1CC0, 0x1CC7}, {0x1CD3, 0x1CD3}, {0x2010, 0x2027}, {0x2030, 0x2043}, {0x2045, 0x2051}, {0x2053, 0x205E}, {0x207D, 0x207E}, {0x208D, 0x208E}, {0x2308, 0x230B}, {0x2329, 0x232A}, {0x2768, 0x2775}, {0x27C5, 0x27C6}, {0x27E6, 0x27EF}, {0x2983, 0x2998}, {0x29D8, 0x29DB}, {0x29FC, 0x29FD}, {0x2CF9, 0x2CFC}, {0x2CFE, 0x2CFF}, {0x2D70, 0x2D70}, {0x2E00, 0x2E2E}, {0x2E30, 0x2E4F}, {0x2E52, 0x2E52}, {0x3001, 0x3003}, {0x3008, 0x3011}, {0x3014, 0x301F}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30A0, 0x30A0}, {0x30FB, 0x30FB}, {0xA4FE, 0xA4FF}, {0xA60D, 0xA60F}, {0xA673, 0xA673}, {0xA67E, 0xA67E}, {0xA6F2, 0xA6F7}, {0xA874, 0xA877}, {0xA8CE, 0xA8CF}, {0xA8F8, 0xA8FA}, {0xA8FC, 0xA8FC}, {0xA92E, 0xA92F}, {0xA95F, 0xA95F}, {0xA9C1, 0xA9CD}, {0xA9DE, 0xA9DF}, {0xAA5C, 0xAA5F}, {0xAADE, 0xAADF}, {0xAAF0, 0xAAF1}, {0xABEB, 0xABEB}, {0xFD3E, 0xFD3F}, {0xFE10, 0xFE19}, {0xFE30, 0xFE52}, {0xFE54, 0xFE61}, {0xFE63, 0xFE63}, {0xFE68, 0xFE68}, {0xFE6A, 0xFE6B}, {0xFF01, 0xFF03}, {0xFF05, 0xFF0A}, {0xFF0C, 0xFF0F}, {0xFF1A, 0xFF1B}, {0xFF1F, 0xFF20}, {0xFF3B, 0xFF3D}, {0xFF3F, 0xFF3F}, {0xFF5B, 0xFF5B}, {0xFF5D, 0xFF5D}, {0xFF5F, 0xFF65}, {0x10100, 0x10102}, {0x1039F, 0x1039F}, {0x103D0, 0x103D0}, {0x1056F, 0x1056F}, {0x10857, 0x10857}, {0x1091F, 0x1091F}, {0x1093F, 0x1093F}, {0x10A50, 0x10A58}, {0x10A7F, 0x10A7F}, {0x10AF0, 0x10AF6}, {0x10B39, 0x10B3F}, {0x10B99, 0x10B9C}, {0x10EAD, 0x10EAD}, {0x10F55, 0x10F59}, {0x11047, 0x1104D}, {0x110BB, 0x110BC}, {0x110BE, 0x110C1}, {0x11140, 0x11143}, {0x11174, 0x11175}, {0x111C5, 0x111C8}, {0x111CD, 0x111CD}, {0x111DB, 0x111DB}, {0x111DD, 0x111DF}, {0x11238, 0x1123D}, {0x112A9, 0x112A9}, {0x1144B, 0x1144F}, {0x1145A, 0x1145B}, {0x1145D, 0x1145D}, {0x114C6, 0x114C6}, {0x115C1, 0x115D7}, {0x11641, 0x11643}, {0x11660, 0x1166C}, {0x1173C, 0x1173E}, {0x1183B, 0x1183B}, {0x11944, 0x11946}, {0x119E2, 0x119E2}, {0x11A3F, 0x11A46}, {0x11A9A, 0x11A9C}, {0x11A9E, 0x11AA2}, {0x11C41, 0x11C45}, {0x11C70, 0x11C71}, {0x11EF7, 0x11EF8}, {0x11FFF, 0x11FFF}, {0x12470, 0x12474}, {0x16A6E, 0x16A6F}, {0x16AF5, 0x16AF5}, {0x16B37, 0x16B3B}, {0x16B44, 0x16B44}, {0x16E97, 0x16E9A}, {0x16FE2, 0x16FE2}, {0x1BC9F, 0x1BC9F}, {0x1DA87, 0x1DA8B}, {0x1E95E, 0x1E95F}, }; + +static const std::vector> symbol_ranges = { +{0x24, 0x24}, {0x2B, 0x2B}, {0x3C, 0x3E}, {0x5E, 0x5E}, {0x60, 0x60}, {0x7C, 0x7C}, {0x7E, 0x7E}, {0xA2, 0xA6}, {0xA8, 0xA9}, {0xAC, 0xAC}, {0xAE, 0xB1}, {0xB4, 0xB4}, {0xB8, 0xB8}, {0xD7, 0xD7}, {0xF7, 0xF7}, {0x2C2, 0x2C5}, {0x2D2, 0x2DF}, {0x2E5, 0x2EB}, {0x2ED, 0x2ED}, {0x2EF, 0x2FF}, {0x375, 0x375}, {0x384, 0x385}, {0x3F6, 0x3F6}, {0x482, 0x482}, {0x58D, 0x58F}, {0x606, 0x608}, {0x60B, 0x60B}, {0x60E, 0x60F}, {0x6DE, 0x6DE}, {0x6E9, 0x6E9}, {0x6FD, 0x6FE}, {0x7F6, 0x7F6}, {0x7FE, 0x7FF}, {0x9F2, 0x9F3}, {0x9FA, 0x9FB}, {0xAF1, 0xAF1}, {0xB70, 0xB70}, {0xBF3, 0xBFA}, {0xC7F, 0xC7F}, {0xD4F, 0xD4F}, {0xD79, 0xD79}, {0xE3F, 0xE3F}, {0xF01, 0xF03}, {0xF13, 0xF13}, {0xF15, 0xF17}, {0xF1A, 0xF1F}, {0xF34, 0xF34}, {0xF36, 0xF36}, {0xF38, 0xF38}, {0xFBE, 0xFC5}, {0xFC7, 0xFCC}, {0xFCE, 0xFCF}, {0xFD5, 0xFD8}, {0x109E, 0x109F}, {0x1390, 0x1399}, {0x166D, 0x166D}, {0x17DB, 0x17DB}, {0x1940, 0x1940}, {0x19DE, 0x19FF}, {0x1B61, 0x1B6A}, {0x1B74, 0x1B7C}, {0x1FBD, 0x1FBD}, {0x1FBF, 0x1FC1}, {0x1FCD, 0x1FCF}, {0x1FDD, 0x1FDF}, {0x1FED, 0x1FEF}, {0x1FFD, 0x1FFE}, {0x2044, 0x2044}, {0x2052, 0x2052}, {0x207A, 0x207C}, {0x208A, 0x208C}, {0x20A0, 0x20BF}, {0x2100, 0x2101}, {0x2103, 0x2106}, {0x2108, 0x2109}, {0x2114, 0x2114}, {0x2116, 0x2118}, {0x211E, 0x2123}, {0x2125, 0x2125}, {0x2127, 0x2127}, {0x2129, 0x2129}, {0x212E, 0x212E}, {0x213A, 0x213B}, {0x2140, 0x2144}, {0x214A, 0x214D}, {0x214F, 0x214F}, {0x218A, 0x218B}, {0x2190, 0x2307}, {0x230C, 0x2328}, {0x232B, 0x2426}, {0x2440, 0x244A}, {0x249C, 0x24E9}, {0x2500, 0x2767}, {0x2794, 0x27C4}, {0x27C7, 0x27E5}, {0x27F0, 0x2982}, {0x2999, 0x29D7}, {0x29DC, 0x29FB}, {0x29FE, 0x2B73}, {0x2B76, 0x2B95}, {0x2B97, 0x2BFF}, {0x2CE5, 0x2CEA}, {0x2E50, 0x2E51}, {0x2E80, 0x2E99}, {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, {0x3004, 0x3004}, {0x3012, 0x3013}, {0x3020, 0x3020}, {0x3036, 0x3037}, {0x303E, 0x303F}, {0x309B, 0x309C}, {0x3190, 0x3191}, {0x3196, 0x319F}, {0x31C0, 0x31E3}, {0x3200, 0x321E}, {0x322A, 0x3247}, {0x3250, 0x3250}, {0x3260, 0x327F}, {0x328A, 0x32B0}, {0x32C0, 0x33FF}, {0x4DC0, 0x4DFF}, {0xA490, 0xA4C6}, {0xA700, 0xA716}, {0xA720, 0xA721}, {0xA789, 0xA78A}, {0xA828, 0xA82B}, {0xA836, 0xA839}, {0xAA77, 0xAA79}, {0xAB5B, 0xAB5B}, {0xAB6A, 0xAB6B}, {0xFB29, 0xFB29}, {0xFBB2, 0xFBC1}, {0xFDFC, 0xFDFD}, {0xFE62, 0xFE62}, {0xFE64, 0xFE66}, {0xFE69, 0xFE69}, {0xFF04, 0xFF04}, {0xFF0B, 0xFF0B}, {0xFF1C, 0xFF1E}, {0xFF3E, 0xFF3E}, {0xFF40, 0xFF40}, {0xFF5C, 0xFF5C}, {0xFF5E, 0xFF5E}, {0xFFE0, 0xFFE6}, {0xFFE8, 0xFFEE}, {0xFFFC, 0xFFFD}, {0x10137, 0x1013F}, {0x10179, 0x10189}, {0x1018C, 0x1018E}, {0x10190, 0x1019C}, {0x101A0, 0x101A0}, {0x101D0, 0x101FC}, {0x10877, 0x10878}, {0x10AC8, 0x10AC8}, {0x1173F, 0x1173F}, {0x11FD5, 0x11FF1}, {0x16B3C, 0x16B3F}, {0x16B45, 0x16B45}, {0x1BC9C, 0x1BC9C}, {0x1D000, 0x1D0F5}, {0x1D100, 0x1D126}, {0x1D129, 0x1D164}, {0x1D16A, 0x1D16C}, {0x1D183, 0x1D184}, {0x1D18C, 0x1D1A9}, {0x1D1AE, 0x1D1E8}, {0x1D200, 0x1D241}, {0x1D245, 0x1D245}, {0x1D300, 0x1D356}, {0x1D6C1, 0x1D6C1}, {0x1D6DB, 0x1D6DB}, {0x1D6FB, 0x1D6FB}, {0x1D715, 0x1D715}, {0x1D735, 0x1D735}, {0x1D74F, 0x1D74F}, {0x1D76F, 0x1D76F}, {0x1D789, 0x1D789}, {0x1D7A9, 0x1D7A9}, {0x1D7C3, 0x1D7C3}, {0x1D800, 0x1D9FF}, {0x1DA37, 0x1DA3A}, {0x1DA6D, 0x1DA74}, {0x1DA76, 0x1DA83}, {0x1DA85, 0x1DA86}, {0x1E14F, 0x1E14F}, {0x1E2FF, 0x1E2FF}, {0x1ECAC, 0x1ECAC}, {0x1ECB0, 0x1ECB0}, {0x1ED2E, 0x1ED2E}, {0x1EEF0, 0x1EEF1}, {0x1F000, 0x1F02B}, {0x1F030, 0x1F093}, {0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF}, {0x1F0C1, 0x1F0CF}, {0x1F0D1, 0x1F0F5}, {0x1F10D, 0x1F1AD}, {0x1F1E6, 0x1F202}, {0x1F210, 0x1F23B}, {0x1F240, 0x1F248}, {0x1F250, 0x1F251}, {0x1F260, 0x1F265}, {0x1F300, 0x1F6D7}, {0x1F6E0, 0x1F6EC}, {0x1F6F0, 0x1F6FC}, {0x1F700, 0x1F773}, {0x1F780, 0x1F7D8}, {0x1F7E0, 0x1F7EB}, {0x1F800, 0x1F80B}, {0x1F810, 0x1F847}, {0x1F850, 0x1F859}, {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, {0x1F8B0, 0x1F8B1}, {0x1F900, 0x1F978}, {0x1F97A, 0x1F9CB}, {0x1F9CD, 0x1FA53}, {0x1FA60, 0x1FA6D}, {0x1FA70, 0x1FA74}, {0x1FA78, 0x1FA7A}, {0x1FA80, 0x1FA86}, {0x1FA90, 0x1FAA8}, {0x1FAB0, 0x1FAB6}, {0x1FAC0, 0x1FAC2}, {0x1FAD0, 0x1FAD6}, {0x1FB00, 0x1FB92}, {0x1FB94, 0x1FBCA}, }; + +static const std::vector> control_ranges = { +{0x0, 0x8}, {0xE, 0x1B}, {0x7F, 0x84}, {0x86, 0x9F}, {0xAD, 0xAD}, {0x378, 0x379}, {0x380, 0x383}, {0x38B, 0x38B}, {0x38D, 0x38D}, {0x3A2, 0x3A2}, {0x530, 0x530}, {0x557, 0x558}, {0x58B, 0x58C}, {0x590, 0x590}, {0x5C8, 0x5CF}, {0x5EB, 0x5EE}, {0x5F5, 0x605}, {0x61C, 0x61D}, {0x6DD, 0x6DD}, {0x70E, 0x70F}, {0x74B, 0x74C}, {0x7B2, 0x7BF}, {0x7FB, 0x7FC}, {0x82E, 0x82F}, {0x83F, 0x83F}, {0x85C, 0x85D}, {0x85F, 0x85F}, {0x86B, 0x89F}, {0x8B5, 0x8B5}, {0x8C8, 0x8D2}, {0x8E2, 0x8E2}, {0x984, 0x984}, {0x98D, 0x98E}, {0x991, 0x992}, {0x9A9, 0x9A9}, {0x9B1, 0x9B1}, {0x9B3, 0x9B5}, {0x9BA, 0x9BB}, {0x9C5, 0x9C6}, {0x9C9, 0x9CA}, {0x9CF, 0x9D6}, {0x9D8, 0x9DB}, {0x9DE, 0x9DE}, {0x9E4, 0x9E5}, {0x9FF, 0xA00}, {0xA04, 0xA04}, {0xA0B, 0xA0E}, {0xA11, 0xA12}, {0xA29, 0xA29}, {0xA31, 0xA31}, {0xA34, 0xA34}, {0xA37, 0xA37}, {0xA3A, 0xA3B}, {0xA3D, 0xA3D}, {0xA43, 0xA46}, {0xA49, 0xA4A}, {0xA4E, 0xA50}, {0xA52, 0xA58}, {0xA5D, 0xA5D}, {0xA5F, 0xA65}, {0xA77, 0xA80}, {0xA84, 0xA84}, {0xA8E, 0xA8E}, {0xA92, 0xA92}, {0xAA9, 0xAA9}, {0xAB1, 0xAB1}, {0xAB4, 0xAB4}, {0xABA, 0xABB}, {0xAC6, 0xAC6}, {0xACA, 0xACA}, {0xACE, 0xACF}, {0xAD1, 0xADF}, {0xAE4, 0xAE5}, {0xAF2, 0xAF8}, {0xB00, 0xB00}, {0xB04, 0xB04}, {0xB0D, 0xB0E}, {0xB11, 0xB12}, {0xB29, 0xB29}, {0xB31, 0xB31}, {0xB34, 0xB34}, {0xB3A, 0xB3B}, {0xB45, 0xB46}, {0xB49, 0xB4A}, {0xB4E, 0xB54}, {0xB58, 0xB5B}, {0xB5E, 0xB5E}, {0xB64, 0xB65}, {0xB78, 0xB81}, {0xB84, 0xB84}, {0xB8B, 0xB8D}, {0xB91, 0xB91}, {0xB96, 0xB98}, {0xB9B, 0xB9B}, {0xB9D, 0xB9D}, {0xBA0, 0xBA2}, {0xBA5, 0xBA7}, {0xBAB, 0xBAD}, {0xBBA, 0xBBD}, {0xBC3, 0xBC5}, {0xBC9, 0xBC9}, {0xBCE, 0xBCF}, {0xBD1, 0xBD6}, {0xBD8, 0xBE5}, {0xBFB, 0xBFF}, {0xC0D, 0xC0D}, {0xC11, 0xC11}, {0xC29, 0xC29}, {0xC3A, 0xC3C}, {0xC45, 0xC45}, {0xC49, 0xC49}, {0xC4E, 0xC54}, {0xC57, 0xC57}, {0xC5B, 0xC5F}, {0xC64, 0xC65}, {0xC70, 0xC76}, {0xC8D, 0xC8D}, {0xC91, 0xC91}, {0xCA9, 0xCA9}, {0xCB4, 0xCB4}, {0xCBA, 0xCBB}, {0xCC5, 0xCC5}, {0xCC9, 0xCC9}, {0xCCE, 0xCD4}, {0xCD7, 0xCDD}, {0xCDF, 0xCDF}, {0xCE4, 0xCE5}, {0xCF0, 0xCF0}, {0xCF3, 0xCFF}, {0xD0D, 0xD0D}, {0xD11, 0xD11}, {0xD45, 0xD45}, {0xD49, 0xD49}, {0xD50, 0xD53}, {0xD64, 0xD65}, {0xD80, 0xD80}, {0xD84, 0xD84}, {0xD97, 0xD99}, {0xDB2, 0xDB2}, {0xDBC, 0xDBC}, {0xDBE, 0xDBF}, {0xDC7, 0xDC9}, {0xDCB, 0xDCE}, {0xDD5, 0xDD5}, {0xDD7, 0xDD7}, {0xDE0, 0xDE5}, {0xDF0, 0xDF1}, {0xDF5, 0xE00}, {0xE3B, 0xE3E}, {0xE5C, 0xE80}, {0xE83, 0xE83}, {0xE85, 0xE85}, {0xE8B, 0xE8B}, {0xEA4, 0xEA4}, {0xEA6, 0xEA6}, {0xEBE, 0xEBF}, {0xEC5, 0xEC5}, {0xEC7, 0xEC7}, {0xECE, 0xECF}, {0xEDA, 0xEDB}, {0xEE0, 0xEFF}, {0xF48, 0xF48}, {0xF6D, 0xF70}, {0xF98, 0xF98}, {0xFBD, 0xFBD}, {0xFCD, 0xFCD}, {0xFDB, 0xFFF}, {0x10C6, 0x10C6}, {0x10C8, 0x10CC}, {0x10CE, 0x10CF}, {0x1249, 0x1249}, {0x124E, 0x124F}, {0x1257, 0x1257}, {0x1259, 0x1259}, {0x125E, 0x125F}, {0x1289, 0x1289}, {0x128E, 0x128F}, {0x12B1, 0x12B1}, {0x12B6, 0x12B7}, {0x12BF, 0x12BF}, {0x12C1, 0x12C1}, {0x12C6, 0x12C7}, {0x12D7, 0x12D7}, {0x1311, 0x1311}, {0x1316, 0x1317}, {0x135B, 0x135C}, {0x137D, 0x137F}, {0x139A, 0x139F}, {0x13F6, 0x13F7}, {0x13FE, 0x13FF}, {0x169D, 0x169F}, {0x16F9, 0x16FF}, {0x170D, 0x170D}, {0x1715, 0x171F}, {0x1737, 0x173F}, {0x1754, 0x175F}, {0x176D, 0x176D}, {0x1771, 0x1771}, {0x1774, 0x177F}, {0x17DE, 0x17DF}, {0x17EA, 0x17EF}, {0x17FA, 0x17FF}, {0x180E, 0x180F}, {0x181A, 0x181F}, {0x1879, 0x187F}, {0x18AB, 0x18AF}, {0x18F6, 0x18FF}, {0x191F, 0x191F}, {0x192C, 0x192F}, {0x193C, 0x193F}, {0x1941, 0x1943}, {0x196E, 0x196F}, {0x1975, 0x197F}, {0x19AC, 0x19AF}, {0x19CA, 0x19CF}, {0x19DB, 0x19DD}, {0x1A1C, 0x1A1D}, {0x1A5F, 0x1A5F}, {0x1A7D, 0x1A7E}, {0x1A8A, 0x1A8F}, {0x1A9A, 0x1A9F}, {0x1AAE, 0x1AAF}, {0x1AC1, 0x1AFF}, {0x1B4C, 0x1B4F}, {0x1B7D, 0x1B7F}, {0x1BF4, 0x1BFB}, {0x1C38, 0x1C3A}, {0x1C4A, 0x1C4C}, {0x1C89, 0x1C8F}, {0x1CBB, 0x1CBC}, {0x1CC8, 0x1CCF}, {0x1CFB, 0x1CFF}, {0x1DFA, 0x1DFA}, {0x1F16, 0x1F17}, {0x1F1E, 0x1F1F}, {0x1F46, 0x1F47}, {0x1F4E, 0x1F4F}, {0x1F58, 0x1F58}, {0x1F5A, 0x1F5A}, {0x1F5C, 0x1F5C}, {0x1F5E, 0x1F5E}, {0x1F7E, 0x1F7F}, {0x1FB5, 0x1FB5}, {0x1FC5, 0x1FC5}, {0x1FD4, 0x1FD5}, {0x1FDC, 0x1FDC}, {0x1FF0, 0x1FF1}, {0x1FF5, 0x1FF5}, {0x1FFF, 0x1FFF}, {0x200B, 0x200F}, {0x202A, 0x202E}, {0x2060, 0x206F}, {0x2072, 0x2073}, {0x208F, 0x208F}, {0x209D, 0x209F}, {0x20C0, 0x20CF}, {0x20F1, 0x20FF}, {0x218C, 0x218F}, {0x2427, 0x243F}, {0x244B, 0x245F}, {0x2B74, 0x2B75}, {0x2B96, 0x2B96}, {0x2C2F, 0x2C2F}, {0x2C5F, 0x2C5F}, {0x2CF4, 0x2CF8}, {0x2D26, 0x2D26}, {0x2D28, 0x2D2C}, {0x2D2E, 0x2D2F}, {0x2D68, 0x2D6E}, {0x2D71, 0x2D7E}, {0x2D97, 0x2D9F}, {0x2DA7, 0x2DA7}, {0x2DAF, 0x2DAF}, {0x2DB7, 0x2DB7}, {0x2DBF, 0x2DBF}, {0x2DC7, 0x2DC7}, {0x2DCF, 0x2DCF}, {0x2DD7, 0x2DD7}, {0x2DDF, 0x2DDF}, {0x2E53, 0x2E7F}, {0x2E9A, 0x2E9A}, {0x2EF4, 0x2EFF}, {0x2FD6, 0x2FEF}, {0x2FFC, 0x2FFF}, {0x3040, 0x3040}, {0x3097, 0x3098}, {0x3100, 0x3104}, {0x3130, 0x3130}, {0x318F, 0x318F}, {0x31E4, 0x31EF}, {0x321F, 0x321F}, {0x9FFD, 0x9FFF}, {0xA48D, 0xA48F}, {0xA4C7, 0xA4CF}, {0xA62C, 0xA63F}, {0xA6F8, 0xA6FF}, {0xA7C0, 0xA7C1}, {0xA7CB, 0xA7F4}, {0xA82D, 0xA82F}, {0xA83A, 0xA83F}, {0xA878, 0xA87F}, {0xA8C6, 0xA8CD}, {0xA8DA, 0xA8DF}, {0xA954, 0xA95E}, {0xA97D, 0xA97F}, {0xA9CE, 0xA9CE}, {0xA9DA, 0xA9DD}, {0xA9FF, 0xA9FF}, {0xAA37, 0xAA3F}, {0xAA4E, 0xAA4F}, {0xAA5A, 0xAA5B}, {0xAAC3, 0xAADA}, {0xAAF7, 0xAB00}, {0xAB07, 0xAB08}, {0xAB0F, 0xAB10}, {0xAB17, 0xAB1F}, {0xAB27, 0xAB27}, {0xAB2F, 0xAB2F}, {0xAB6C, 0xAB6F}, {0xABEE, 0xABEF}, {0xABFA, 0xABFF}, {0xD7A4, 0xD7AF}, {0xD7C7, 0xD7CA}, {0xD7FC, 0xF8FF}, {0xFA6E, 0xFA6F}, {0xFADA, 0xFAFF}, {0xFB07, 0xFB12}, {0xFB18, 0xFB1C}, {0xFB37, 0xFB37}, {0xFB3D, 0xFB3D}, {0xFB3F, 0xFB3F}, {0xFB42, 0xFB42}, {0xFB45, 0xFB45}, {0xFBC2, 0xFBD2}, {0xFD40, 0xFD4F}, {0xFD90, 0xFD91}, {0xFDC8, 0xFDEF}, {0xFDFE, 0xFDFF}, {0xFE1A, 0xFE1F}, {0xFE53, 0xFE53}, {0xFE67, 0xFE67}, {0xFE6C, 0xFE6F}, {0xFE75, 0xFE75}, {0xFEFD, 0xFF00}, {0xFFBF, 0xFFC1}, {0xFFC8, 0xFFC9}, {0xFFD0, 0xFFD1}, {0xFFD8, 0xFFD9}, {0xFFDD, 0xFFDF}, {0xFFE7, 0xFFE7}, {0xFFEF, 0xFFFB}, {0xFFFE, 0xFFFF}, {0x1000C, 0x1000C}, {0x10027, 0x10027}, {0x1003B, 0x1003B}, {0x1003E, 0x1003E}, {0x1004E, 0x1004F}, {0x1005E, 0x1007F}, {0x100FB, 0x100FF}, {0x10103, 0x10106}, {0x10134, 0x10136}, {0x1018F, 0x1018F}, {0x1019D, 0x1019F}, {0x101A1, 0x101CF}, {0x101FE, 0x1027F}, {0x1029D, 0x1029F}, {0x102D1, 0x102DF}, {0x102FC, 0x102FF}, {0x10324, 0x1032C}, {0x1034B, 0x1034F}, {0x1037B, 0x1037F}, {0x1039E, 0x1039E}, {0x103C4, 0x103C7}, {0x103D6, 0x103FF}, {0x1049E, 0x1049F}, {0x104AA, 0x104AF}, {0x104D4, 0x104D7}, {0x104FC, 0x104FF}, {0x10528, 0x1052F}, {0x10564, 0x1056E}, {0x10570, 0x105FF}, {0x10737, 0x1073F}, {0x10756, 0x1075F}, {0x10768, 0x107FF}, {0x10806, 0x10807}, {0x10809, 0x10809}, {0x10836, 0x10836}, {0x10839, 0x1083B}, {0x1083D, 0x1083E}, {0x10856, 0x10856}, {0x1089F, 0x108A6}, {0x108B0, 0x108DF}, {0x108F3, 0x108F3}, {0x108F6, 0x108FA}, {0x1091C, 0x1091E}, {0x1093A, 0x1093E}, {0x10940, 0x1097F}, {0x109B8, 0x109BB}, {0x109D0, 0x109D1}, {0x10A04, 0x10A04}, {0x10A07, 0x10A0B}, {0x10A14, 0x10A14}, {0x10A18, 0x10A18}, {0x10A36, 0x10A37}, {0x10A3B, 0x10A3E}, {0x10A49, 0x10A4F}, {0x10A59, 0x10A5F}, {0x10AA0, 0x10ABF}, {0x10AE7, 0x10AEA}, {0x10AF7, 0x10AFF}, {0x10B36, 0x10B38}, {0x10B56, 0x10B57}, {0x10B73, 0x10B77}, {0x10B92, 0x10B98}, {0x10B9D, 0x10BA8}, {0x10BB0, 0x10BFF}, {0x10C49, 0x10C7F}, {0x10CB3, 0x10CBF}, {0x10CF3, 0x10CF9}, {0x10D28, 0x10D2F}, {0x10D3A, 0x10E5F}, {0x10E7F, 0x10E7F}, {0x10EAA, 0x10EAA}, {0x10EAE, 0x10EAF}, {0x10EB2, 0x10EFF}, {0x10F28, 0x10F2F}, {0x10F5A, 0x10FAF}, {0x10FCC, 0x10FDF}, {0x10FF7, 0x10FFF}, {0x1104E, 0x11051}, {0x11070, 0x1107E}, {0x110BD, 0x110BD}, {0x110C2, 0x110CF}, {0x110E9, 0x110EF}, {0x110FA, 0x110FF}, {0x11135, 0x11135}, {0x11148, 0x1114F}, {0x11177, 0x1117F}, {0x111E0, 0x111E0}, {0x111F5, 0x111FF}, {0x11212, 0x11212}, {0x1123F, 0x1127F}, {0x11287, 0x11287}, {0x11289, 0x11289}, {0x1128E, 0x1128E}, {0x1129E, 0x1129E}, {0x112AA, 0x112AF}, {0x112EB, 0x112EF}, {0x112FA, 0x112FF}, {0x11304, 0x11304}, {0x1130D, 0x1130E}, {0x11311, 0x11312}, {0x11329, 0x11329}, {0x11331, 0x11331}, {0x11334, 0x11334}, {0x1133A, 0x1133A}, {0x11345, 0x11346}, {0x11349, 0x1134A}, {0x1134E, 0x1134F}, {0x11351, 0x11356}, {0x11358, 0x1135C}, {0x11364, 0x11365}, {0x1136D, 0x1136F}, {0x11375, 0x113FF}, {0x1145C, 0x1145C}, {0x11462, 0x1147F}, {0x114C8, 0x114CF}, {0x114DA, 0x1157F}, {0x115B6, 0x115B7}, {0x115DE, 0x115FF}, {0x11645, 0x1164F}, {0x1165A, 0x1165F}, {0x1166D, 0x1167F}, {0x116B9, 0x116BF}, {0x116CA, 0x116FF}, {0x1171B, 0x1171C}, {0x1172C, 0x1172F}, {0x11740, 0x117FF}, {0x1183C, 0x1189F}, {0x118F3, 0x118FE}, {0x11907, 0x11908}, {0x1190A, 0x1190B}, {0x11914, 0x11914}, {0x11917, 0x11917}, {0x11936, 0x11936}, {0x11939, 0x1193A}, {0x11947, 0x1194F}, {0x1195A, 0x1199F}, {0x119A8, 0x119A9}, {0x119D8, 0x119D9}, {0x119E5, 0x119FF}, {0x11A48, 0x11A4F}, {0x11AA3, 0x11ABF}, {0x11AF9, 0x11BFF}, {0x11C09, 0x11C09}, {0x11C37, 0x11C37}, {0x11C46, 0x11C4F}, {0x11C6D, 0x11C6F}, {0x11C90, 0x11C91}, {0x11CA8, 0x11CA8}, {0x11CB7, 0x11CFF}, {0x11D07, 0x11D07}, {0x11D0A, 0x11D0A}, {0x11D37, 0x11D39}, {0x11D3B, 0x11D3B}, {0x11D3E, 0x11D3E}, {0x11D48, 0x11D4F}, {0x11D5A, 0x11D5F}, {0x11D66, 0x11D66}, {0x11D69, 0x11D69}, {0x11D8F, 0x11D8F}, {0x11D92, 0x11D92}, {0x11D99, 0x11D9F}, {0x11DAA, 0x11EDF}, {0x11EF9, 0x11FAF}, {0x11FB1, 0x11FBF}, {0x11FF2, 0x11FFE}, {0x1239A, 0x123FF}, {0x1246F, 0x1246F}, {0x12475, 0x1247F}, {0x12544, 0x12FFF}, {0x1342F, 0x143FF}, {0x14647, 0x167FF}, {0x16A39, 0x16A3F}, {0x16A5F, 0x16A5F}, {0x16A6A, 0x16A6D}, {0x16A70, 0x16ACF}, {0x16AEE, 0x16AEF}, {0x16AF6, 0x16AFF}, {0x16B46, 0x16B4F}, {0x16B5A, 0x16B5A}, {0x16B62, 0x16B62}, {0x16B78, 0x16B7C}, {0x16B90, 0x16E3F}, {0x16E9B, 0x16EFF}, {0x16F4B, 0x16F4E}, {0x16F88, 0x16F8E}, {0x16FA0, 0x16FDF}, {0x16FE5, 0x16FEF}, {0x16FF2, 0x16FFF}, {0x187F8, 0x187FF}, {0x18CD6, 0x18CFF}, {0x18D09, 0x1AFFF}, {0x1B11F, 0x1B14F}, {0x1B153, 0x1B163}, {0x1B168, 0x1B16F}, {0x1B2FC, 0x1BBFF}, {0x1BC6B, 0x1BC6F}, {0x1BC7D, 0x1BC7F}, {0x1BC89, 0x1BC8F}, {0x1BC9A, 0x1BC9B}, {0x1BCA0, 0x1CFFF}, {0x1D0F6, 0x1D0FF}, {0x1D127, 0x1D128}, {0x1D173, 0x1D17A}, {0x1D1E9, 0x1D1FF}, {0x1D246, 0x1D2DF}, {0x1D2F4, 0x1D2FF}, {0x1D357, 0x1D35F}, {0x1D379, 0x1D3FF}, {0x1D455, 0x1D455}, {0x1D49D, 0x1D49D}, {0x1D4A0, 0x1D4A1}, {0x1D4A3, 0x1D4A4}, {0x1D4A7, 0x1D4A8}, {0x1D4AD, 0x1D4AD}, {0x1D4BA, 0x1D4BA}, {0x1D4BC, 0x1D4BC}, {0x1D4C4, 0x1D4C4}, {0x1D506, 0x1D506}, {0x1D50B, 0x1D50C}, {0x1D515, 0x1D515}, {0x1D51D, 0x1D51D}, {0x1D53A, 0x1D53A}, {0x1D53F, 0x1D53F}, {0x1D545, 0x1D545}, {0x1D547, 0x1D549}, {0x1D551, 0x1D551}, {0x1D6A6, 0x1D6A7}, {0x1D7CC, 0x1D7CD}, {0x1DA8C, 0x1DA9A}, {0x1DAA0, 0x1DAA0}, {0x1DAB0, 0x1DFFF}, {0x1E007, 0x1E007}, {0x1E019, 0x1E01A}, {0x1E022, 0x1E022}, {0x1E025, 0x1E025}, {0x1E02B, 0x1E0FF}, {0x1E12D, 0x1E12F}, {0x1E13E, 0x1E13F}, {0x1E14A, 0x1E14D}, {0x1E150, 0x1E2BF}, {0x1E2FA, 0x1E2FE}, {0x1E300, 0x1E7FF}, {0x1E8C5, 0x1E8C6}, {0x1E8D7, 0x1E8FF}, {0x1E94C, 0x1E94F}, {0x1E95A, 0x1E95D}, {0x1E960, 0x1EC70}, {0x1ECB5, 0x1ED00}, {0x1ED3E, 0x1EDFF}, {0x1EE04, 0x1EE04}, {0x1EE20, 0x1EE20}, {0x1EE23, 0x1EE23}, {0x1EE25, 0x1EE26}, {0x1EE28, 0x1EE28}, {0x1EE33, 0x1EE33}, {0x1EE38, 0x1EE38}, {0x1EE3A, 0x1EE3A}, {0x1EE3C, 0x1EE41}, {0x1EE43, 0x1EE46}, {0x1EE48, 0x1EE48}, {0x1EE4A, 0x1EE4A}, {0x1EE4C, 0x1EE4C}, {0x1EE50, 0x1EE50}, {0x1EE53, 0x1EE53}, {0x1EE55, 0x1EE56}, {0x1EE58, 0x1EE58}, {0x1EE5A, 0x1EE5A}, {0x1EE5C, 0x1EE5C}, {0x1EE5E, 0x1EE5E}, {0x1EE60, 0x1EE60}, {0x1EE63, 0x1EE63}, {0x1EE65, 0x1EE66}, {0x1EE6B, 0x1EE6B}, {0x1EE73, 0x1EE73}, {0x1EE78, 0x1EE78}, {0x1EE7D, 0x1EE7D}, {0x1EE7F, 0x1EE7F}, {0x1EE8A, 0x1EE8A}, {0x1EE9C, 0x1EEA0}, {0x1EEA4, 0x1EEA4}, {0x1EEAA, 0x1EEAA}, {0x1EEBC, 0x1EEEF}, {0x1EEF2, 0x1EFFF}, {0x1F02C, 0x1F02F}, {0x1F094, 0x1F09F}, {0x1F0AF, 0x1F0B0}, {0x1F0C0, 0x1F0C0}, {0x1F0D0, 0x1F0D0}, {0x1F0F6, 0x1F0FF}, {0x1F1AE, 0x1F1E5}, {0x1F203, 0x1F20F}, {0x1F23C, 0x1F23F}, {0x1F249, 0x1F24F}, {0x1F252, 0x1F25F}, {0x1F266, 0x1F2FF}, {0x1F6D8, 0x1F6DF}, {0x1F6ED, 0x1F6EF}, {0x1F6FD, 0x1F6FF}, {0x1F774, 0x1F77F}, {0x1F7D9, 0x1F7DF}, {0x1F7EC, 0x1F7FF}, {0x1F80C, 0x1F80F}, {0x1F848, 0x1F84F}, {0x1F85A, 0x1F85F}, {0x1F888, 0x1F88F}, {0x1F8AE, 0x1F8AF}, {0x1F8B2, 0x1F8FF}, {0x1F979, 0x1F979}, {0x1F9CC, 0x1F9CC}, {0x1FA54, 0x1FA5F}, {0x1FA6E, 0x1FA6F}, {0x1FA75, 0x1FA77}, {0x1FA7B, 0x1FA7F}, {0x1FA87, 0x1FA8F}, {0x1FAA9, 0x1FAAF}, {0x1FAB7, 0x1FABF}, {0x1FAC3, 0x1FACF}, {0x1FAD7, 0x1FAFF}, {0x1FB93, 0x1FB93}, {0x1FBCB, 0x1FBEF}, {0x1FBFA, 0x1FFFF}, {0x2A6DE, 0x2A6FF}, {0x2B735, 0x2B73F}, {0x2B81E, 0x2B81F}, {0x2CEA2, 0x2CEAF}, {0x2EBE1, 0x2F7FF}, {0x2FA1E, 0x2FFFF}, {0x3134B, 0xE00FF}, {0xE01F0, 0x10FFFF}, }; + +//String +bool CNCTString::operator==(const std::string& other) const { + return str.compare(other) == 0; +} +bool CNCTString::operator==(const char other) const { + return str.compare(std::string(1, other)) == 0; +} +bool CNCTString::operator==(const CNCTString& other) const { + return str.compare(other.str) == 0; +} +// + operators +CNCTString& CNCTString::operator+=(const std::string& other) { + str += other; + int new_len = CNCTUnicode::strlen_utf8(other); + utf8_chars += new_len; + char_type = CNCTUnicode::string_identify(str); + seq_offset_bytes += other.size(); + seq_offset_utf8_chars += new_len; + return *this; +} + +CNCTString& CNCTString::operator+=(const char other) { + std::string str = std::string(1, other); + *this += str; + return *this; +} + +CNCTString& CNCTString::operator+=(const CNCTString& other) { + str += other.str; + utf8_chars += other.utf8_chars; + char_type = CNCTUnicode::string_identify(str); + seq_offset_bytes += other.str.size(); + seq_offset_utf8_chars += other.utf8_chars; + return *this; +} + +struct CRCompare { + bool operator()(const std::pair& p, int i) { + return p.second < i; + } + bool operator()(int i, const std::pair& p) { + return i < p.first; + } +}; + +// binary search for code range +bool CNCTUnicode::check_code_range(int c, const std::vector> &ranges) +{ + auto it = std::upper_bound(ranges.begin(), ranges.end(), c, CRCompare()); + if (it != ranges.begin()) { + --it; + } + return c >= it->first && c <= it->second; +} + +// these are binary searches, it takes only a few operations +CNCTCharType CNCTUnicode::get_code_type(int c) +{ + if (check_code_range(c, letter_ranges)) + return LETTER; + if (check_code_range(c, digit_ranges)) + return DIGIT; + if (check_code_range(c, whitespace_ranges)) + return WHITESPACE; + if (check_code_range(c, punctuation_ranges)) + return PUNCTUATION; + if (check_code_range(c, symbol_ranges)) + return SYMBOL; + if (check_code_range(c, accent_mark_ranges)) + return ACCENT_MARK; + if (check_code_range(c, control_ranges)) + return CONTROL; + return UNIDENTIFIED; +} + +static int utf8_to_unicode(const std::string& utf8_char) { + int c = 0; + int len = (int)utf8_char.size(); + if (len == 1) { + c = utf8_char[0]; + } else if (len == 2) { + c = ((utf8_char[0] & 0x1F) << 6) | (utf8_char[1] & 0x3F); + } else if (len == 3) { + c = ((utf8_char[0] & 0x0F) << 12) | ((utf8_char[1] & 0x3F) << 6) | (utf8_char[2] & 0x3F); + } else if (len == 4) { + c = ((utf8_char[0] & 0x07) << 18) | ((utf8_char[1] & 0x3F) << 12) | ((utf8_char[2] & 0x3F) << 6) | (utf8_char[3] & 0x3F); + } + return c; +} + +CNCTCharType CNCTUnicode::get_code_type(const std::string &utf8_char) +{ + return get_code_type(utf8_to_unicode(utf8_char)); +} + +int CNCTUnicode::utf8_len(const char c) +{ + if ((c & 0x80) == 0) + return 1; // ASCII character + if ((c & 0xE0) == 0xC0) + return 2; // 2-byte character + if ((c & 0xF0) == 0xE0) + return 3; // 3-byte character + if ((c & 0xF0) == 0xF0) + return 4; // 4-byte character + return 1; // not valid utf8 + // static const uint8_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; + // return lookup[static_cast(c) >> 4]; +} + +int CNCTUnicode::strlen_utf8(const std::string src) +{ + int len = 0; + for (std::string::const_iterator it = src.begin(); it != src.end(); ++it) + { + int char_len = utf8_len(*it); + if (char_len > 1) + { + it += char_len - 1; + } + len += 1; + } + return len; +} + +// split a string into unicode strings +std::vector CNCTUnicode::split_utf8(const std::string &src) +{ + std::vector result; + for (std::string::const_iterator it = src.begin(); it != src.end(); ++it) + { + int char_len = utf8_len(*it); + std::string str(it, it + char_len); + result.push_back(str); + if (char_len > 1) + { + it += char_len - 1; + } + } + return result; +} + +// split a string into unicode strings (CNCTString) with sequence information +std::vector CNCTUnicode::split_utf8_enhanced(const std::string &src) +{ + std::vector result; + int seq_offset_bytes=0; + int seq_offset_utf8_chars=0; + for (std::string::const_iterator it = src.begin(); it != src.end(); ++it) + { + int char_len = utf8_len(*it); + std::string str(it, it + char_len); + CNCTString cnct_str; + cnct_str.seq_offset_bytes = seq_offset_bytes; + cnct_str.seq_offset_utf8_chars = seq_offset_utf8_chars; + cnct_str.str = str; + cnct_str.utf8_chars = 1; + cnct_str.char_type = get_code_type(str); + #if 0 + switch (cnct_str.char_type) + { + case DIGIT: + printf("%s = DIGIT\n", str.c_str()); + break; + case LETTER: + printf("%s = LETTER\n", str.c_str()); + break; + case WHITESPACE: + printf("%s = WHITESPACE\n", str.c_str()); + break; + case PUNCTUATION: + printf("%s = PUNCTUATION\n", str.c_str()); + break; + case UNIDENTIFIED: + printf("%s = UNIDENTIFIED\n", str.c_str()); + break; + case SYMBOL: + printf("%s = SYMBOL\n", str.c_str()); + break; + case CONTROL: + printf("%s = CONTROL\n", str.c_str()); + break; + } + #endif + + + result.push_back(cnct_str); + seq_offset_bytes += char_len; + seq_offset_utf8_chars += 1; + if (char_len > 1) + { + it += char_len - 1; + } + + } + return result; +} + +// return the type of the string +CNCTCharType CNCTUnicode::string_identify(const std::string &str) +{ + CNCTCharType result = UNIDENTIFIED; + std::string::const_iterator it = str.begin(); + while (it != str.end()) + { + int len = utf8_len(*it); + int c = 0; + for (int i = 0; i < len && it != str.end(); ++i, ++it) + { + c = (c << 8) | static_cast(*it); + } + switch (get_code_type(c)) + { + case DIGIT: + if (result == UNIDENTIFIED) + { + result = DIGIT; + } + else if (result != DIGIT) + { + return MIXED; + } + break; + case LETTER: + if (result == UNIDENTIFIED) + { + result = LETTER; + } + else if (result != LETTER) + { + return MIXED; + } + break; + case WHITESPACE: + if (result == UNIDENTIFIED) + { + result = WHITESPACE; + } + else if (result != WHITESPACE) + { + return MIXED; + } + break; + case PUNCTUATION: + if (result == UNIDENTIFIED) + { + result = PUNCTUATION; + } + else if (result != PUNCTUATION) + { + return MIXED; + } + break; + default: + return MIXED; + break; + } + } + return result; +} + +// verify the content of a string +bool CNCTUnicode::string_test(const std::string &str, CNCTCharType chartype) +{ + std::string::const_iterator it = str.begin(); + while (it != str.end()) + { + int len = utf8_len(*it); + int c = 0; + for (int i = 0; i < len && it != str.end(); ++i, ++it) + { + c = (c << 8) | static_cast(*it); + } + if (get_code_type(c) != chartype) + { + return false; + } + } + return true; +} + +// Ported from libfalcon.cpp (https://github.com/cmp-nct/ggllm.cpp) + +std::string replaceAll(std::string str, const std::string& from, const std::string& to) { + size_t start_pos = 0; + while((start_pos = str.find(from, start_pos)) != std::string::npos) { + str.replace(start_pos, from.length(), to); + start_pos += to.length(); // Handles case where 'to' is a substring of 'from' + } + return str; +} + +struct TrieNode { + std::map map; + int32_t Id = -1; +}; + +struct Trie { + TrieNode *root; + + Trie() : root(new TrieNode()) {} + ~Trie() { + if(root) + deleteTrie(root); + } + // Move constructor + Trie(Trie&& other) noexcept : root(other.root) { + other.root = nullptr; + } + + // Move assignment operator + Trie& operator=(Trie&& other) noexcept { + if (this != &other) { + if(root) + deleteTrie(root); + root = other.root; + other.root = nullptr; + } + return *this; + } + + void insert(const std::string &token, int32_t Id) { + TrieNode* current = root; + for(auto ch : token) { + if(current->map.find(ch) == current->map.end()) + current->map[ch] = new TrieNode(); + current = current->map[ch]; + } + current->Id = Id; + } + + void reset() + { + deleteTrie(root); + root = new TrieNode(); + } + +private: + void deleteTrie(TrieNode* node) { + for(auto &it: node->map) { + deleteTrie(it.second); + } + delete node; + } + +}; + +struct gpt2bpe_vocab { + using id = int32_t; + using token = std::string; + std::map max_token_length; // max length, for each 2byte prefix + + std::map, int> bpe_ranks; + std::vector> bpe_merges; + std::map special_tokens; + + id special_bos_id = 0; + id special_eos_id = 0; + id special_unk_id = 0; + id special_sep_id = 0; + id special_pad_id = 0; + + bool special_have_bos = false; + bool special_have_eos = false; + bool special_have_unk = false; + bool special_have_sep = false; + bool special_have_pad = false; + + std::unordered_map token_to_id; + std::unordered_map id_to_token; + + Trie trie; // highspeed access to tokens by prefix tree + + // populate trie from map + void populate_trie_from_map() { + trie.reset(); + for (const auto& pair : token_to_id) { + trie.insert(pair.first, pair.second); + if (pair.first.size() >= 2) + { + std::string prefix = pair.first.substr(0, 2); + max_token_length[prefix] = std::max(max_token_length[prefix], (uint32_t)pair.first.size()); + } + } + } + // populate token ranks map + int populate_bpe_ranks(std::vector> bpe_merges_) { + for (int i = 0; i < (int)bpe_merges_.size(); i++) { + bpe_ranks.emplace(bpe_merges_[i], i); + } + bpe_merges = bpe_merges_; + + // populate special tokens too (0-11 and if available 65024++) + + #if 0 + for (int i = 0; i < 12; i++) { + special_tokens[id_to_token[i].tok] = i; + } + for (int i = 65024; i < (int)id_to_token.size(); i++) { + special_tokens[id_to_token[i].tok] = i; + } + #endif + + // token_to_id[""] = 11; // bugfix for TII instruct training (blocks stopwords) + // special_tokens[""] = 11; // bugfix for TII instruct training (blocks stopwords) + + + return bpe_merges_.size(); + } + + // Trim whitespace characters from the beginning and end of the string + void trim(std::string& str) { + // Remove whitespace characters from the beginning of the string + str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](int ch) { + return !std::isspace(ch); + })); + + // Remove whitespace characters from the end of the string + str.erase(std::find_if(str.rbegin(), str.rend(), [](int ch) { + return !std::isspace(ch); + }).base(), str.end()); + } + + // removed, merges loaded from gguf model file: + // requires the standard HF type tokenizer.json (pretty printed) + // std::vector> parse_json_to_bpe_merges(const std::string& filename) { + + // get max token length available for a prefix of 2 bytes (string at least 2 bytes long) + int get_max_token_length(const std::string& string) const { + if (string.size() < 2) + return -1; + std::string prefix = string.substr(0, 2); + if (max_token_length.find(prefix) == max_token_length.end()) + return 0; + return max_token_length.at(prefix); + } + + // function to find if two tokens match in bpe_rank, return rank or -1 + int find_bpe_rank(const std::string& token1, const std::string& token2) const + { + std::string left_token = token1; + std::string right_token = token2; + left_token = replaceAll(left_token, " ", "Ġ"); + left_token = replaceAll(left_token, "\n", "Ċ"); + right_token = replaceAll(right_token, " ", "Ġ"); + right_token = replaceAll(right_token, "\n", "Ċ"); + + auto it = bpe_ranks.find(std::make_pair(left_token, right_token)); + if (it == bpe_ranks.end()) + return -1; + return it->second; + } + + std::pair find_longest_match(const std::string& snippet) const { + TrieNode* current = trie.root; + gpt2bpe_vocab::id last_matched_id = -1; + std::string last_matched_token = ""; + std::string current_token = ""; + for (auto ch : snippet) { + if (current->map.find(ch) == current->map.end()) { + break; + } + current = current->map[ch]; + current_token += ch; + if (current->Id != -1) { + last_matched_id = current->Id; + last_matched_token = current_token; + } + } + return {last_matched_id, last_matched_token}; + } + +}; + + +// +// tokenizer - bpe type, gpt2 tokenization compatible +// + +struct ggllm_bpe_symbol { + using index = int; + index prev; + index next; + const char * text; + size_t n; +}; + +static_assert(std::is_trivially_copyable::value, "ggllm_bpe_symbol is not trivially copyable"); + +struct ggllm_bpe_bigram { + struct comparator + { + bool operator()(ggllm_bpe_bigram & l, ggllm_bpe_bigram & r) + { + return l.rank > r.rank || (l.rank == r.rank && l.left > r.left); + } + }; + + using queue_storage = std::vector; + using queue = std::priority_queue; + ggllm_bpe_symbol::index left; + ggllm_bpe_symbol::index right; + std::string text; + int rank; + size_t size; +}; + +struct gpt2bpe_tokenizer { + gpt2bpe_tokenizer(const gpt2bpe_vocab & vocab, bool g2ws_): vocab_(vocab) { flag_g2ws = g2ws_; } + + void tokenize(const std::string & text, std::vector & output) { + int final_prev_index = -1; + // auto start = ggml_time_us(); + auto word_collection = bpe_gpt2_preprocess(text); + // auto end = ggml_time_us(); + // fprintf(stderr, "%s: preprocessing took %0.3f ms\n", __func__, (end - start) / 1000.0); + + + symbols_final.clear(); + + for (auto & word : word_collection) + { + work_queue_ = ggllm_bpe_bigram::queue(); + symbols_.clear(); + bool is_special = false; + for (auto it = vocab_.special_tokens.begin(); it != vocab_.special_tokens.end(); ++it) + { + std::string special_token = it->first; + if (word.compare(special_token) == 0) + { + ggllm_bpe_symbol sym; + sym.text = word.c_str(); + sym.n = word.size(); + sym.prev = -1; + sym.next = -1; + symbols_.emplace_back(sym); + is_special = true; + break; + } + } + + int index = 0; + size_t offset = 0; + if (!is_special) + { + + while (offset < word.size()) + { + ggllm_bpe_symbol sym; + size_t char_len = std::min(word.size() - offset, (size_t) CNCTUnicode::utf8_len(word[offset])); + sym.text = word.c_str() + offset; + sym.n = 1; + sym.n = char_len; + offset += sym.n; + sym.prev = index - 1; + sym.next = offset == word.size() ? -1 : index + 1; + index++; + symbols_.emplace_back(sym); + } + for (size_t i = 1; i < symbols_.size(); ++i) { + add_new_bigram(i - 1, i); + } + } + // build token(s) + while (!work_queue_.empty()) + { + auto bigram = work_queue_.top(); + work_queue_.pop(); + + auto & left_symbol = symbols_[bigram.left]; + auto & right_symbol = symbols_[bigram.right]; + + if (left_symbol.n == 0 || right_symbol.n == 0) { + continue; + } + std::string left_token = std::string(left_symbol.text, left_symbol.n); + std::string right_token = std::string(right_symbol.text, right_symbol.n); + if (left_token + right_token != bigram.text) { + continue; // Skip this bigram if it's outdated + } + + // merge the right sym into the left one + left_symbol.n += right_symbol.n; + right_symbol.n = 0; + + // remove the right sym from the chain + left_symbol.next = right_symbol.next; + if (right_symbol.next >= 0) { + symbols_[right_symbol.next].prev = bigram.left; + } + + add_new_bigram(left_symbol.prev, bigram.left); // left side of current symbol + add_new_bigram(bigram.left, left_symbol.next); // right side of current symbol + } + // add the fnished tokens to the final list keeping correct order for next and prev + + for (auto & sym : symbols_) + { + if (sym.n > 0) + { + sym.prev = final_prev_index; + sym.next = -1; + if (final_prev_index != -1) + { + symbols_final[final_prev_index].next = symbols_final.size(); + } + symbols_final.emplace_back(sym); + final_prev_index = symbols_final.size() - 1; + } + } + } + + + symbols_ = symbols_final; + if (symbols_.size()) + for (int i = 0; i != -1; i = symbols_[i].next) { + auto & symbol = symbols_[i]; + if (symbol.n == 0) { + continue; + } + std::string str = std::string(symbol.text, symbol.n); + std::string str_decoded = decode_token(str); + auto token = vocab_.token_to_id.find(str_decoded); + + if (token == vocab_.token_to_id.end()) { + for (auto j = str_decoded.begin(); j != str_decoded.end(); ++j) { + std::string byte_str(1, *j); + auto token_multibyte = vocab_.token_to_id.find(byte_str); + if (token_multibyte == vocab_.token_to_id.end()) { + fprintf(stderr,"ERROR: byte not found in vocab: '%s'\n", byte_str.c_str()); + } + output.push_back((*token_multibyte).second); + } + } else { + output.push_back((*token).second); + } + } + } + +private: + void add_new_bigram(int left, int right) + { + if (left == -1 || right == -1) return; + + std::string left_token = std::string(symbols_[left].text, symbols_[left].n); + std::string right_token = std::string(symbols_[right].text, symbols_[right].n); + + int rank_found = -1; + rank_found = vocab_.find_bpe_rank(left_token, right_token); + + if (rank_found < 0) { + return; + } + + ggllm_bpe_bigram bigram; + bigram.left = left; + bigram.right = right; + bigram.rank = rank_found; + bigram.size = left_token.size() + right_token.size(); + bigram.text = left_token + right_token; + work_queue_.push(bigram); + } + + std::unordered_map bytes_to_unicode() { + static std::unordered_map hex_map = { { 0x21, "\x21" }, { 0x22, "\x22" }, { 0x23, "\x23" }, { 0x24, "\x24" }, { 0x25, "\x25" }, { 0x26, "\x26" }, { 0x27, "\x27" }, { 0x28, "\x28" }, { 0x29, "\x29" }, { 0x2A, "\x2A" }, { 0x2B, "\x2B" }, { 0x2C, "\x2C" }, { 0x2D, "\x2D" }, { 0x2E, "\x2E" }, { 0x2F, "\x2F" }, { 0x30, "\x30" }, { 0x31, "\x31" }, { 0x32, "\x32" }, { 0x33, "\x33" }, { 0x34, "\x34" }, { 0x35, "\x35" }, { 0x36, "\x36" }, { 0x37, "\x37" }, { 0x38, "\x38" }, { 0x39, "\x39" }, { 0x3A, "\x3A" }, { 0x3B, "\x3B" }, { 0x3C, "\x3C" }, { 0x3D, "\x3D" }, { 0x3E, "\x3E" }, { 0x3F, "\x3F" }, { 0x40, "\x40" }, { 0x41, "\x41" }, { 0x42, "\x42" }, { 0x43, "\x43" }, { 0x44, "\x44" }, { 0x45, "\x45" }, { 0x46, "\x46" }, { 0x47, "\x47" }, { 0x48, "\x48" }, { 0x49, "\x49" }, { 0x4A, "\x4A" }, { 0x4B, "\x4B" }, { 0x4C, "\x4C" }, { 0x4D, "\x4D" }, { 0x4E, "\x4E" }, { 0x4F, "\x4F" }, { 0x50, "\x50" }, { 0x51, "\x51" }, { 0x52, "\x52" }, { 0x53, "\x53" }, { 0x54, "\x54" }, { 0x55, "\x55" }, { 0x56, "\x56" }, { 0x57, "\x57" }, { 0x58, "\x58" }, { 0x59, "\x59" }, { 0x5A, "\x5A" }, { 0x5B, "\x5B" }, { 0x5C, "\x5C" }, { 0x5D, "\x5D" }, { 0x5E, "\x5E" }, { 0x5F, "\x5F" }, { 0x60, "\x60" }, { 0x61, "\x61" }, { 0x62, "\x62" }, { 0x63, "\x63" }, { 0x64, "\x64" }, { 0x65, "\x65" }, { 0x66, "\x66" }, { 0x67, "\x67" }, { 0x68, "\x68" }, { 0x69, "\x69" }, { 0x6A, "\x6A" }, { 0x6B, "\x6B" }, { 0x6C, "\x6C" }, { 0x6D, "\x6D" }, { 0x6E, "\x6E" }, { 0x6F, "\x6F" }, { 0x70, "\x70" }, { 0x71, "\x71" }, { 0x72, "\x72" }, { 0x73, "\x73" }, { 0x74, "\x74" }, { 0x75, "\x75" }, { 0x76, "\x76" }, { 0x77, "\x77" }, { 0x78, "\x78" }, { 0x79, "\x79" }, { 0x7A, "\x7A" }, { 0x7B, "\x7B" }, { 0x7C, "\x7C" }, { 0x7D, "\x7D" }, { 0x7E, "\x7E" }, { 0xA1, "\xC2\xA1" }, { 0xA2, "\xC2\xA2" }, { 0xA3, "\xC2\xA3" }, { 0xA4, "\xC2\xA4" }, { 0xA5, "\xC2\xA5" }, { 0xA6, "\xC2\xA6" }, { 0xA7, "\xC2\xA7" }, { 0xA8, "\xC2\xA8" }, { 0xA9, "\xC2\xA9" }, { 0xAA, "\xC2\xAA" }, { 0xAB, "\xC2\xAB" }, { 0xAC, "\xC2\xAC" }, { 0xAE, "\xC2\xAE" }, { 0xAF, "\xC2\xAF" }, { 0xB0, "\xC2\xB0" }, { 0xB1, "\xC2\xB1" }, { 0xB2, "\xC2\xB2" }, { 0xB3, "\xC2\xB3" }, { 0xB4, "\xC2\xB4" }, { 0xB5, "\xC2\xB5" }, { 0xB6, "\xC2\xB6" }, { 0xB7, "\xC2\xB7" }, { 0xB8, "\xC2\xB8" }, { 0xB9, "\xC2\xB9" }, { 0xBA, "\xC2\xBA" }, { 0xBB, "\xC2\xBB" }, { 0xBC, "\xC2\xBC" }, { 0xBD, "\xC2\xBD" }, { 0xBE, "\xC2\xBE" }, { 0xBF, "\xC2\xBF" }, { 0xC0, "\xC3\x80" }, { 0xC1, "\xC3\x81" }, { 0xC2, "\xC3\x82" }, { 0xC3, "\xC3\x83" }, { 0xC4, "\xC3\x84" }, { 0xC5, "\xC3\x85" }, { 0xC6, "\xC3\x86" }, { 0xC7, "\xC3\x87" }, { 0xC8, "\xC3\x88" }, { 0xC9, "\xC3\x89" }, { 0xCA, "\xC3\x8A" }, { 0xCB, "\xC3\x8B" }, { 0xCC, "\xC3\x8C" }, { 0xCD, "\xC3\x8D" }, { 0xCE, "\xC3\x8E" }, { 0xCF, "\xC3\x8F" }, { 0xD0, "\xC3\x90" }, { 0xD1, "\xC3\x91" }, { 0xD2, "\xC3\x92" }, { 0xD3, "\xC3\x93" }, { 0xD4, "\xC3\x94" }, { 0xD5, "\xC3\x95" }, { 0xD6, "\xC3\x96" }, { 0xD7, "\xC3\x97" }, { 0xD8, "\xC3\x98" }, { 0xD9, "\xC3\x99" }, { 0xDA, "\xC3\x9A" }, { 0xDB, "\xC3\x9B" }, { 0xDC, "\xC3\x9C" }, { 0xDD, "\xC3\x9D" }, { 0xDE, "\xC3\x9E" }, { 0xDF, "\xC3\x9F" }, { 0xE0, "\xC3\xA0" }, { 0xE1, "\xC3\xA1" }, { 0xE2, "\xC3\xA2" }, { 0xE3, "\xC3\xA3" }, { 0xE4, "\xC3\xA4" }, { 0xE5, "\xC3\xA5" }, { 0xE6, "\xC3\xA6" }, { 0xE7, "\xC3\xA7" }, { 0xE8, "\xC3\xA8" }, { 0xE9, "\xC3\xA9" }, { 0xEA, "\xC3\xAA" }, { 0xEB, "\xC3\xAB" }, { 0xEC, "\xC3\xAC" }, { 0xED, "\xC3\xAD" }, { 0xEE, "\xC3\xAE" }, { 0xEF, "\xC3\xAF" }, { 0xF0, "\xC3\xB0" }, { 0xF1, "\xC3\xB1" }, { 0xF2, "\xC3\xB2" }, { 0xF3, "\xC3\xB3" }, { 0xF4, "\xC3\xB4" }, { 0xF5, "\xC3\xB5" }, { 0xF6, "\xC3\xB6" }, { 0xF7, "\xC3\xB7" }, { 0xF8, "\xC3\xB8" }, { 0xF9, "\xC3\xB9" }, { 0xFA, "\xC3\xBA" }, { 0xFB, "\xC3\xBB" }, { 0xFC, "\xC3\xBC" }, { 0xFD, "\xC3\xBD" }, { 0xFE, "\xC3\xBE" }, { 0xFF, "\xC3\xBF" }, { 0x00, "\xC4\x80" }, { 0x01, "\xC4\x81" }, { 0x02, "\xC4\x82" }, { 0x03, "\xC4\x83" }, { 0x04, "\xC4\x84" }, { 0x05, "\xC4\x85" }, { 0x06, "\xC4\x86" }, { 0x07, "\xC4\x87" }, { 0x08, "\xC4\x88" }, { 0x09, "\xC4\x89" }, { 0x0A, "\xC4\x8A" }, { 0x0B, "\xC4\x8B" }, { 0x0C, "\xC4\x8C" }, { 0x0D, "\xC4\x8D" }, { 0x0E, "\xC4\x8E" }, { 0x0F, "\xC4\x8F" }, { 0x10, "\xC4\x90" }, { 0x11, "\xC4\x91" }, { 0x12, "\xC4\x92" }, { 0x13, "\xC4\x93" }, { 0x14, "\xC4\x94" }, { 0x15, "\xC4\x95" }, { 0x16, "\xC4\x96" }, { 0x17, "\xC4\x97" }, { 0x18, "\xC4\x98" }, { 0x19, "\xC4\x99" }, { 0x1A, "\xC4\x9A" }, { 0x1B, "\xC4\x9B" }, { 0x1C, "\xC4\x9C" }, { 0x1D, "\xC4\x9D" }, { 0x1E, "\xC4\x9E" }, { 0x1F, "\xC4\x9F" }, { 0x20, "\xC4\xA0" }, { 0x7F, "\xC4\xA1" }, { 0x80, "\xC4\xA2" }, { 0x81, "\xC4\xA3" }, { 0x82, "\xC4\xA4" }, { 0x83, "\xC4\xA5" }, { 0x84, "\xC4\xA6" }, { 0x85, "\xC4\xA7" }, { 0x86, "\xC4\xA8" }, { 0x87, "\xC4\xA9" }, { 0x88, "\xC4\xAA" }, { 0x89, "\xC4\xAB" }, { 0x8A, "\xC4\xAC" }, { 0x8B, "\xC4\xAD" }, { 0x8C, "\xC4\xAE" }, { 0x8D, "\xC4\xAF" }, { 0x8E, "\xC4\xB0" }, { 0x8F, "\xC4\xB1" }, { 0x90, "\xC4\xB2" }, { 0x91, "\xC4\xB3" }, { 0x92, "\xC4\xB4" }, { 0x93, "\xC4\xB5" }, { 0x94, "\xC4\xB6" }, { 0x95, "\xC4\xB7" }, { 0x96, "\xC4\xB8" }, { 0x97, "\xC4\xB9" }, { 0x98, "\xC4\xBA" }, { 0x99, "\xC4\xBB" }, { 0x9A, "\xC4\xBC" }, { 0x9B, "\xC4\xBD" }, { 0x9C, "\xC4\xBE" }, { 0x9D, "\xC4\xBF" }, { 0x9E, "\xC5\x80" }, { 0x9F, "\xC5\x81" }, { 0xA0, "\xC5\x82" }, { 0xAD, "\xC5\x83" }}; + return hex_map; + } + + std::unordered_map unicode_to_bytes() { + static std::unordered_map hex_map = { { "\x21", 0x21 }, { "\x22", 0x22 }, { "\x23", 0x23 }, { "\x24", 0x24 }, { "\x25", 0x25 }, { "\x26", 0x26 }, { "\x27", 0x27 }, { "\x28", 0x28 }, { "\x29", 0x29 }, { "\x2A", 0x2A }, { "\x2B", 0x2B }, { "\x2C", 0x2C }, { "\x2D", 0x2D }, { "\x2E", 0x2E }, { "\x2F", 0x2F }, { "\x30", 0x30 }, { "\x31", 0x31 }, { "\x32", 0x32 }, { "\x33", 0x33 }, { "\x34", 0x34 }, { "\x35", 0x35 }, { "\x36", 0x36 }, { "\x37", 0x37 }, { "\x38", 0x38 }, { "\x39", 0x39 }, { "\x3A", 0x3A }, { "\x3B", 0x3B }, { "\x3C", 0x3C }, { "\x3D", 0x3D }, { "\x3E", 0x3E }, { "\x3F", 0x3F }, { "\x40", 0x40 }, { "\x41", 0x41 }, { "\x42", 0x42 }, { "\x43", 0x43 }, { "\x44", 0x44 }, { "\x45", 0x45 }, { "\x46", 0x46 }, { "\x47", 0x47 }, { "\x48", 0x48 }, { "\x49", 0x49 }, { "\x4A", 0x4A }, { "\x4B", 0x4B }, { "\x4C", 0x4C }, { "\x4D", 0x4D }, { "\x4E", 0x4E }, { "\x4F", 0x4F }, { "\x50", 0x50 }, { "\x51", 0x51 }, { "\x52", 0x52 }, { "\x53", 0x53 }, { "\x54", 0x54 }, { "\x55", 0x55 }, { "\x56", 0x56 }, { "\x57", 0x57 }, { "\x58", 0x58 }, { "\x59", 0x59 }, { "\x5A", 0x5A }, { "\x5B", 0x5B }, { "\x5C", 0x5C }, { "\x5D", 0x5D }, { "\x5E", 0x5E }, { "\x5F", 0x5F }, { "\x60", 0x60 }, { "\x61", 0x61 }, { "\x62", 0x62 }, { "\x63", 0x63 }, { "\x64", 0x64 }, { "\x65", 0x65 }, { "\x66", 0x66 }, { "\x67", 0x67 }, { "\x68", 0x68 }, { "\x69", 0x69 }, { "\x6A", 0x6A }, { "\x6B", 0x6B }, { "\x6C", 0x6C }, { "\x6D", 0x6D }, { "\x6E", 0x6E }, { "\x6F", 0x6F }, { "\x70", 0x70 }, { "\x71", 0x71 }, { "\x72", 0x72 }, { "\x73", 0x73 }, { "\x74", 0x74 }, { "\x75", 0x75 }, { "\x76", 0x76 }, { "\x77", 0x77 }, { "\x78", 0x78 }, { "\x79", 0x79 }, { "\x7A", 0x7A }, { "\x7B", 0x7B }, { "\x7C", 0x7C }, { "\x7D", 0x7D }, { "\x7E", 0x7E }, { "\xC2\xA1", 0xA1 }, { "\xC2\xA2", 0xA2 }, { "\xC2\xA3", 0xA3 }, { "\xC2\xA4", 0xA4 }, { "\xC2\xA5", 0xA5 }, { "\xC2\xA6", 0xA6 }, { "\xC2\xA7", 0xA7 }, { "\xC2\xA8", 0xA8 }, { "\xC2\xA9", 0xA9 }, { "\xC2\xAA", 0xAA }, { "\xC2\xAB", 0xAB }, { "\xC2\xAC", 0xAC }, { "\xC2\xAE", 0xAE }, { "\xC2\xAF", 0xAF }, { "\xC2\xB0", 0xB0 }, { "\xC2\xB1", 0xB1 }, { "\xC2\xB2", 0xB2 }, { "\xC2\xB3", 0xB3 }, { "\xC2\xB4", 0xB4 }, { "\xC2\xB5", 0xB5 }, { "\xC2\xB6", 0xB6 }, { "\xC2\xB7", 0xB7 }, { "\xC2\xB8", 0xB8 }, { "\xC2\xB9", 0xB9 }, { "\xC2\xBA", 0xBA }, { "\xC2\xBB", 0xBB }, { "\xC2\xBC", 0xBC }, { "\xC2\xBD", 0xBD }, { "\xC2\xBE", 0xBE }, { "\xC2\xBF", 0xBF }, { "\xC3\x80", 0xC0 }, { "\xC3\x81", 0xC1 }, { "\xC3\x82", 0xC2 }, { "\xC3\x83", 0xC3 }, { "\xC3\x84", 0xC4 }, { "\xC3\x85", 0xC5 }, { "\xC3\x86", 0xC6 }, { "\xC3\x87", 0xC7 }, { "\xC3\x88", 0xC8 }, { "\xC3\x89", 0xC9 }, { "\xC3\x8A", 0xCA }, { "\xC3\x8B", 0xCB }, { "\xC3\x8C", 0xCC }, { "\xC3\x8D", 0xCD }, { "\xC3\x8E", 0xCE }, { "\xC3\x8F", 0xCF }, { "\xC3\x90", 0xD0 }, { "\xC3\x91", 0xD1 }, { "\xC3\x92", 0xD2 }, { "\xC3\x93", 0xD3 }, { "\xC3\x94", 0xD4 }, { "\xC3\x95", 0xD5 }, { "\xC3\x96", 0xD6 }, { "\xC3\x97", 0xD7 }, { "\xC3\x98", 0xD8 }, { "\xC3\x99", 0xD9 }, { "\xC3\x9A", 0xDA }, { "\xC3\x9B", 0xDB }, { "\xC3\x9C", 0xDC }, { "\xC3\x9D", 0xDD }, { "\xC3\x9E", 0xDE }, { "\xC3\x9F", 0xDF }, { "\xC3\xA0", 0xE0 }, { "\xC3\xA1", 0xE1 }, { "\xC3\xA2", 0xE2 }, { "\xC3\xA3", 0xE3 }, { "\xC3\xA4", 0xE4 }, { "\xC3\xA5", 0xE5 }, { "\xC3\xA6", 0xE6 }, { "\xC3\xA7", 0xE7 }, { "\xC3\xA8", 0xE8 }, { "\xC3\xA9", 0xE9 }, { "\xC3\xAA", 0xEA }, { "\xC3\xAB", 0xEB }, { "\xC3\xAC", 0xEC }, { "\xC3\xAD", 0xED }, { "\xC3\xAE", 0xEE }, { "\xC3\xAF", 0xEF }, { "\xC3\xB0", 0xF0 }, { "\xC3\xB1", 0xF1 }, { "\xC3\xB2", 0xF2 }, { "\xC3\xB3", 0xF3 }, { "\xC3\xB4", 0xF4 }, { "\xC3\xB5", 0xF5 }, { "\xC3\xB6", 0xF6 }, { "\xC3\xB7", 0xF7 }, { "\xC3\xB8", 0xF8 }, { "\xC3\xB9", 0xF9 }, { "\xC3\xBA", 0xFA }, { "\xC3\xBB", 0xFB }, { "\xC3\xBC", 0xFC }, { "\xC3\xBD", 0xFD }, { "\xC3\xBE", 0xFE }, { "\xC3\xBF", 0xFF }, { "\xC4\x80", 0x00 }, { "\xC4\x81", 0x01 }, { "\xC4\x82", 0x02 }, { "\xC4\x83", 0x03 }, { "\xC4\x84", 0x04 }, { "\xC4\x85", 0x05 }, { "\xC4\x86", 0x06 }, { "\xC4\x87", 0x07 }, { "\xC4\x88", 0x08 }, { "\xC4\x89", 0x09 }, { "\xC4\x8A", 0x0A }, { "\xC4\x8B", 0x0B }, { "\xC4\x8C", 0x0C }, { "\xC4\x8D", 0x0D }, { "\xC4\x8E", 0x0E }, { "\xC4\x8F", 0x0F }, { "\xC4\x90", 0x10 }, { "\xC4\x91", 0x11 }, { "\xC4\x92", 0x12 }, { "\xC4\x93", 0x13 }, { "\xC4\x94", 0x14 }, { "\xC4\x95", 0x15 }, { "\xC4\x96", 0x16 }, { "\xC4\x97", 0x17 }, { "\xC4\x98", 0x18 }, { "\xC4\x99", 0x19 }, { "\xC4\x9A", 0x1A }, { "\xC4\x9B", 0x1B }, { "\xC4\x9C", 0x1C }, { "\xC4\x9D", 0x1D }, { "\xC4\x9E", 0x1E }, { "\xC4\x9F", 0x1F }, { "\xC4\xA0", 0x20 }, { "\xC4\xA1", 0x7F }, { "\xC4\xA2", 0x80 }, { "\xC4\xA3", 0x81 }, { "\xC4\xA4", 0x82 }, { "\xC4\xA5", 0x83 }, { "\xC4\xA6", 0x84 }, { "\xC4\xA7", 0x85 }, { "\xC4\xA8", 0x86 }, { "\xC4\xA9", 0x87 }, { "\xC4\xAA", 0x88 }, { "\xC4\xAB", 0x89 }, { "\xC4\xAC", 0x8A }, { "\xC4\xAD", 0x8B }, { "\xC4\xAE", 0x8C }, { "\xC4\xAF", 0x8D }, { "\xC4\xB0", 0x8E }, { "\xC4\xB1", 0x8F }, { "\xC4\xB2", 0x90 }, { "\xC4\xB3", 0x91 }, { "\xC4\xB4", 0x92 }, { "\xC4\xB5", 0x93 }, { "\xC4\xB6", 0x94 }, { "\xC4\xB7", 0x95 }, { "\xC4\xB8", 0x96 }, { "\xC4\xB9", 0x97 }, { "\xC4\xBA", 0x98 }, { "\xC4\xBB", 0x99 }, { "\xC4\xBC", 0x9A }, { "\xC4\xBD", 0x9B }, { "\xC4\xBE", 0x9C }, { "\xC4\xBF", 0x9D }, { "\xC5\x80", 0x9E }, { "\xC5\x81", 0x9F }, { "\xC5\x82", 0xA0 }, { "\xC5\x83", 0xAD }}; + return hex_map; + } + + // len must be available + bool inline str_is_equal(const char* str1, const char* str2, size_t len) { + for (size_t i = 0; i < len; ++i) { + if (str1[i] != str2[i]) { + return false; + } + } + return true; + } + + std::vector bpe_gpt2_preprocess(const std::string& text) + { + static std::unordered_map< unsigned char, std::string> byte_encoder = bytes_to_unicode(); + std::vector bpe_words; + std::vector bpe_encoded_words; + + + std::string token=""; + const char *raw_text_p = text.c_str(); + // GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+ + bool collecting_numeric = false; + bool collecting_letter = false; + bool collecting_special = false; + bool collecting_whitespace_lookahead = false; + bool collecting=false; + + std::vector text_utf; + text_utf.reserve(text.size()); + bpe_words.reserve(text.size()); + bpe_encoded_words.reserve(text.size()); + + text_utf = CNCTUnicode::split_utf8_enhanced(text); + std::map special_tokens = vocab_.special_tokens; + int smallest_len_special_tokens = 0; + if (special_tokens.size()) + { + smallest_len_special_tokens = special_tokens.begin()->first.size(); + for (auto it = special_tokens.begin(); it != special_tokens.end(); ++it) + { + if (it->first.size() < (size_t)smallest_len_special_tokens) + smallest_len_special_tokens = it->first.size(); + } + } + + for (int i = 0; i < (int)text_utf.size(); i++) + { + const CNCTString &utf_char = text_utf[i]; + bool split_condition = false; + const char *text_pos = raw_text_p + utf_char.seq_offset_bytes; + int bytes_remain = strlen(text_pos); + // forward backward lookups + const CNCTString &utf_char_next = (i+1 < (int)text_utf.size()) ? text_utf[i+1] : CNCTString(); + const CNCTString &utf_char_next_next = (i+2 < (int)text_utf.size()) ? text_utf[i+2] : CNCTString(); + // const CNCTString &utf_char_prev = (i > 0) ? text_utf[i-1] : CNCTString(); + + // handling special tokens + bool special_token_found = false; + if (bytes_remain >= (int)smallest_len_special_tokens) + for (auto it = special_tokens.begin(); it != special_tokens.end(); ++it) + { + if ((bytes_remain) < (int)it->first.size()) + continue; + + if (str_is_equal(text_pos, it->first.c_str(), it->first.size())) + { + if (token.size()) + { + bpe_words.emplace_back(token); // push previous content as token + token.clear(); + collecting = false; + collecting_letter = false; + collecting_numeric = false; + collecting_special = false; + collecting_whitespace_lookahead = false; + } + + bpe_words.emplace_back(it->first); // push special token as token + + // we now advance i until the token is fulfilled by the utf_chars + int st_bytes = (int)it->first.size(); + for (;st_bytes;st_bytes -= text_utf[i++].str.size()); + i--; + special_token_found = true; + break; + } + } + + if (special_token_found) continue; + + + // handling contractions + if (!split_condition && bytes_remain >= 2) + { + // 's|'t|'m|'d + if (utf_char == '\'' && (utf_char_next == 's' || utf_char_next == 't' || utf_char_next == 'm' || utf_char_next == 'd')) + split_condition = true; + if (split_condition) + { + if (token.size()) + bpe_words.emplace_back(token); // push previous content as token + token = utf_char.str + utf_char_next.str; + bpe_words.emplace_back(token); + token=""; + i++; + continue; + } + } + if (!split_condition && bytes_remain >= 3) + { + // 're|'ve|'ll + if (utf_char == '\'' && ( + (utf_char_next == 'r' || utf_char_next_next == 'e') || + (utf_char_next == 'v' || utf_char_next_next == 'e') || + (utf_char_next == 'l' || utf_char_next_next == 'l')) + ) + split_condition = true; + if (split_condition) + { + // current token + next token can be defined + if (token.size()) + bpe_words.emplace_back(token); // push previous content as token + token = utf_char.str + utf_char_next.str + utf_char_next_next.str; + bpe_words.emplace_back(token); // the contraction + token=""; + i+=2; + continue; + } + } + + if (!split_condition && !collecting) + { + if (utf_char.char_type == CNCTCharType::LETTER || (!token.size() && utf_char==" " && utf_char_next.char_type == CNCTCharType::LETTER)) + { + collecting_letter = true; + collecting = true; + } + else if (utf_char.char_type == CNCTCharType::DIGIT || (!token.size() && utf_char==" " && utf_char_next.char_type == CNCTCharType::DIGIT)) + { + collecting_numeric = true; + collecting = true; + } + else if ( + ((utf_char.char_type != CNCTCharType::LETTER && utf_char.char_type != CNCTCharType::DIGIT) && (utf_char.char_type != CNCTCharType::WHITESPACE)) || + (!token.size() && utf_char==" " && utf_char_next.char_type != CNCTCharType::LETTER && utf_char_next.char_type != CNCTCharType::DIGIT && utf_char_next.char_type != CNCTCharType::WHITESPACE) + ) + { + collecting_special = true; + collecting = true; + } + else if (utf_char.char_type == CNCTCharType::WHITESPACE && utf_char_next.char_type == CNCTCharType::WHITESPACE) + { + collecting_whitespace_lookahead = true; + collecting = true; + } else if (utf_char.char_type == CNCTCharType::WHITESPACE) + { + split_condition = true; + } + } else + if (!split_condition && collecting) + { + if (collecting_letter && utf_char.char_type != CNCTCharType::LETTER) + { + split_condition = true; + } + else if (collecting_numeric && utf_char.char_type != CNCTCharType::DIGIT) + { + split_condition = true; + } + else if (collecting_special && (utf_char.char_type == CNCTCharType::LETTER || utf_char.char_type == CNCTCharType::DIGIT || utf_char.char_type == CNCTCharType::WHITESPACE)) + { + split_condition = true; + } + else if (collecting_whitespace_lookahead && utf_char_next.char_type != CNCTCharType::WHITESPACE) + { + split_condition = true; + } + } + if(utf_char_next.str.size() == 0) + { + split_condition = true; // final + token += utf_char.str; + } + + if (split_condition) + { + if (token.size()) + bpe_words.emplace_back(token); + token = utf_char.str; + collecting = false; + collecting_letter = false; + collecting_numeric = false; + collecting_special = false; + collecting_whitespace_lookahead = false; + } else token += utf_char.str; + } + + for (std::string& word : bpe_words) + { + std::string encoded_token=""; + for (char& c : word) + { + encoded_token += byte_encoder[c]; + } + bpe_encoded_words.emplace_back(encoded_token); + } + + return bpe_encoded_words; + } + + // decoder (for one token) + std::string decode_token(const std::string& token) + { + static std::unordered_map< std::string, unsigned char> byte_decoder = unicode_to_bytes(); + std::string decoded_token=""; + auto unicode_seqeunces = CNCTUnicode::split_utf8(token); + for (auto& unicode_sequence : unicode_seqeunces) + { + decoded_token += byte_decoder[unicode_sequence]; + + } + + return decoded_token; + } + + + const gpt2bpe_vocab & vocab_; + std::vector symbols_; + std::vector symbols_final; + ggllm_bpe_bigram::queue work_queue_; + bool flag_g2ws=false; +}; + +static std::vector gpt2bpe_tokenize(const gpt2bpe_vocab & vocab, const std::string & text, bool bos, bool g2ws ) { + gpt2bpe_tokenizer tokenizer(vocab, g2ws); + std::vector output; + + if (text.empty()) { + return output; + } + + if (bos && vocab.special_have_bos) { + output.push_back(vocab.special_bos_id); + } + + tokenizer.tokenize(text, output); + return output; +} + + +#endif // CMPNCT_GPT2BPE From fb0b24370563824203f7619615a9d57a33dee57f Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 4 Aug 2023 04:02:10 +0200 Subject: [PATCH 075/242] Makefile : remove gptneox-common --- Makefile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 7d37b66a698ec..90563d7813e67 100644 --- a/Makefile +++ b/Makefile @@ -329,9 +329,6 @@ grammar-parser.o: examples/grammar-parser.cpp examples/grammar-parser.h libllama.so: llama.o ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -shared -fPIC -o $@ $^ $(LDFLAGS) -gptneox-common.o: gptneox-common.cpp gptneox-common.h - $(CXX) $(CXXFLAGS) -c $< -o $@ - clean: rm -vf *.o *.so *.dll main quantize quantize-stats perplexity embedding benchmark-matmult save-load-state server simple vdot train-text-from-scratch embd-input-test gguf build-info.h $(TEST_TARGETS) @@ -376,7 +373,7 @@ embd-input-test: $(LIB_PRE)embdinput$(DSO_EXT) examples/embd-input/embd-input-te gguf: examples/gguf/gguf.cpp build-info.h ggml.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -gptneox-main: gptneox-main.cpp gptneox-common.o ggml.o $(OBJS) +gptneox-main: gptneox-main.cpp ggml.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp build-info.h ggml.o llama.o $(OBJS) From 278ada9572c645a8353edce256453a755ed1743e Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 4 Aug 2023 04:07:57 +0200 Subject: [PATCH 076/242] gguf.py : bytesarray for gpt2bpe tokenizer --- gguf.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gguf.py b/gguf.py index 88e2bee0733b1..1e4f8ea3b235f 100644 --- a/gguf.py +++ b/gguf.py @@ -10,7 +10,7 @@ from typing import Any, IO, List import numpy as np - +import sys class GGMLQuantizationType(IntEnum): F32 = 0 @@ -45,7 +45,7 @@ class GGUFValueType(IntEnum): @staticmethod def get_type(val): - if isinstance(val, str) or isinstance(val, bytes): + if isinstance(val, str) or isinstance(val, bytes) or isinstance(val, bytearray): return GGUFValueType.STRING elif isinstance(val, list): return GGUFValueType.ARRAY @@ -53,8 +53,11 @@ def get_type(val): return GGUFValueType.FLOAT32 elif isinstance(val, bool): return GGUFValueType.BOOL - else: + elif isinstance(val, int): return GGUFValueType.INT32 + else: + print("Unknown type: "+str(type(val))) + sys.exit() class GGUFWriter: From db5618ad9919ca4e4fb29b6fa6509923368f4755 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 4 Aug 2023 04:57:51 +0200 Subject: [PATCH 077/242] cmpnct_gpt2bpe.hpp : comments --- cmpnct_gpt2bpe.hpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cmpnct_gpt2bpe.hpp b/cmpnct_gpt2bpe.hpp index 0743a294ac0ff..ac5f8c672a305 100644 --- a/cmpnct_gpt2bpe.hpp +++ b/cmpnct_gpt2bpe.hpp @@ -11,12 +11,15 @@ #include #include +//----- +// Unicode GPT2 Byte Pair Encoding Tokenizer +// Adapted from https://github.com/cmp-nct/ggllm.cpp +//----- -/** - * https://github.com/cmp-nct/ggllm.cpp - * Minimal library for high performance handling and categorization of UTF8 strings and characters - * Using std::string - */ +// Unicode library (from cmpnct_unicode.cpp) + +// Minimal library for high performance handling and categorization of UTF8 strings and characters +// Using std::string enum CNCTCharType { DIGIT, // a numerical char in any language @@ -367,7 +370,7 @@ bool CNCTUnicode::string_test(const std::string &str, CNCTCharType chartype) return true; } -// Ported from libfalcon.cpp (https://github.com/cmp-nct/ggllm.cpp) +// llama.cpp GPT2 vocab (from libfalcon.cpp) std::string replaceAll(std::string str, const std::string& from, const std::string& to) { size_t start_pos = 0; From 4357e692ace6a5afa6c9a1e2a597795b838e999f Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 7 Aug 2023 13:51:26 +0200 Subject: [PATCH 078/242] gguf.py : use custom alignment if present --- gguf.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gguf.py b/gguf.py index 1e4f8ea3b235f..5eb21ee05fb6f 100644 --- a/gguf.py +++ b/gguf.py @@ -64,6 +64,7 @@ class GGUFWriter: def __init__(self, fout: IO): self.fout = fout self.offset_tensor = 0 + self.data_alignment = constants.GGUF_DEFAULT_ALIGNMENT self.kv_data = b"" self.kv_data_count = 0 self.ti_data = b"" @@ -191,16 +192,17 @@ def add_tensor_info(self, name: str, tensor: np.ndarray): dtype = GGMLQuantizationType.F32 if tensor.dtype == np.float32 else GGMLQuantizationType.F16 self.ti_data += struct.pack(" Date: Mon, 7 Aug 2023 19:02:18 +0300 Subject: [PATCH 079/242] gguf : minor stuff --- examples/gguf/gguf.cpp | 11 +++++------ ggml.c | 4 ---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index 671f8748f5e13..a1b8edc71f373 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -256,10 +256,9 @@ bool gguf_ex_read_0(const std::string & fname) { // find kv string { - char findkey[32]; - sprintf(findkey, "some.parameter.string"); + const char * findkey = "some.parameter.string"; - int keyidx = gguf_find_key(ctx, findkey); + const int keyidx = gguf_find_key(ctx, findkey); if (keyidx == -1) { fprintf(stdout, "%s: find key: %s not found.\n", __func__, findkey); } else { @@ -297,7 +296,7 @@ bool gguf_ex_read_1(const std::string & fname) { }; struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params); - + fprintf(stdout, "%s: version: %d\n", __func__, gguf_get_version(ctx)); fprintf(stdout, "%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx)); fprintf(stdout, "%s: data offset: %zu\n", __func__, gguf_get_data_offset(ctx)); @@ -388,7 +387,7 @@ bool gguf_ex_read_2(const std::string & fname) { // print first 10 elements const float * data = (const float *) cur->data; - + printf("%s data[:10] : ", name); for (int j = 0; j < 10; ++j) { @@ -402,7 +401,7 @@ fprintf(stdout, "%s: ctx_data size: %zu\n", __func__, ggml_get_mem_size(ctx_data ggml_free(ctx_data); gguf_free(ctx); - + return true; } diff --git a/ggml.c b/ggml.c index d3d745333434b..df8dce7e89915 100644 --- a/ggml.c +++ b/ggml.c @@ -19031,10 +19031,6 @@ const char * gguf_get_key(struct gguf_context * ctx, int i) { return ctx->header.kv[i].key.data; } -enum gguf_type gguf_get_type(struct gguf_context * ctx, int i) { - return ctx->header.kv[i].type; -} - const char * gguf_get_arr_str(struct gguf_context * ctx, int key_id, int i) { struct gguf_kv * kv = &ctx->header.kv[key_id]; struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[i]; From 65559a23c8df919bae0ffc82c48c94759b2d53a1 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 7 Aug 2023 22:28:43 +0200 Subject: [PATCH 080/242] Update gptneox-main.cpp --- gptneox-main.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/gptneox-main.cpp b/gptneox-main.cpp index 0161d9c2ee7e8..eecd59678a3eb 100644 --- a/gptneox-main.cpp +++ b/gptneox-main.cpp @@ -443,9 +443,6 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 // load vocab { - - // TODO: implement a better bpe tokenizer, utilizing merges and handles unicode - auto & hparams = model.hparams; int keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.model"); @@ -484,11 +481,6 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 for (size_t i = 0; i < hparams.n_vocab; i++) { std::string word = gguf_get_arr_str(ggufctx, tokens_keyidx, i); - - // TEMP until a better bpe tokenizer is implemented -// word = replace(word, "Ġ", " "); -// word = replace(word, "Ċ", "\n"); - // printf("token %d = '%s'\n",i,word.c_str() ); vocab.token_to_id[word] = i; @@ -1054,7 +1046,6 @@ int main(int argc, char ** argv) { { const int64_t t_start_sample_us = ggml_time_us(); -// id = sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, repeat_last_n, repeat_penalty, rng); id = sample_top_k_top_p_repeat(vocab, logits.data() + (logits.size() - n_vocab), last_n_tokens.data(), last_n_tokens.size(), top_k, top_p, temp, repeat_last_n, repeat_penalty, rng); last_n_tokens.erase(last_n_tokens.begin()); From ece4fc185edf8677fe158a52b52002258b32833b Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Wed, 9 Aug 2023 00:48:33 +0200 Subject: [PATCH 081/242] map tensor names --- gguf_tensor_map.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 gguf_tensor_map.py diff --git a/gguf_tensor_map.py b/gguf_tensor_map.py new file mode 100644 index 0000000000000..4fba633b2bf12 --- /dev/null +++ b/gguf_tensor_map.py @@ -0,0 +1,96 @@ +# Recommended mapping of model tensor names for storage in gguf + +def get_tensor_map( n_blocks : int): + tensor_map = {} + # Token embeddings + mapped_to = "transformer.token_embd" + tensor_map["gpt_neox.embed_in"] = mapped_to # gptneox + tensor_map["transformer.wte"] = mapped_to # gpt2 mpt + tensor_map["transformer.word_embeddings"] = mapped_to # falcon + tensor_map["model.embed_tokens"] = mapped_to # llama-hf + tensor_map["tok_embeddings"] = mapped_to # llama-pth + # Position embeddings + mapped_to = "transformer.pos_embd" + tensor_map["transformer.wpe"] = mapped_to # gpt2 + # Output norm + mapped_to = "transformer.output_norm" + tensor_map["gpt_neox.final_layer_norm"] = mapped_to # gptneox + tensor_map["transformer.ln_f"] = mapped_to # gpt2 falcon + tensor_map["transformer.norm_f"] = mapped_to # mpt + tensor_map["model.norm"] = mapped_to # llama-hf + tensor_map["norm"] = mapped_to # llama-pth + # Output + mapped_to = "transformer.output" + tensor_map["embed_out"] = mapped_to # gptneox + tensor_map["lm_head"] = mapped_to # gpt2 mpt falcon llama-hf + tensor_map["output"] = mapped_to # llama-pth + # Attention and fee-forward layer blocks + for i in range(0,n_blocks): + # Attention norm 1 + mapped_to = "transformer.blocks."+str(i)+".attn_norm_1" + tensor_map["gpt_neox.layers."+str(i)+".input_layernorm"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".ln_1"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".norm_1"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".input_layernorm"] = mapped_to # falcon7b + tensor_map["transformer.h."+str(i)+".ln_attn"] = mapped_to # falcon40b + tensor_map["model.layers."+str(i)+".input_layernorm"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention_norm"] = mapped_to # llama-pth + # Attention norm 2 + mapped_to = "transformer.blocks."+str(i)+".attn_norm_2" + tensor_map["transformer.h."+str(i)+".ln_mlp"] = mapped_to # falcon40b + # Attention query-key-value + mapped_to = "transformer.blocks."+str(i)+".attn_qkv" + tensor_map["gpt_neox.layers."+str(i)+".attention.query_key_value"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".attn.c_attn"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".attn.Wqkv"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".self_attention.query_key_value"] = mapped_to # falcon + # Attention query + mapped_to = "transformer.blocks."+str(i)+".attn_q" + tensor_map["model.layers."+str(i)+".self_attn.q_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention.wq"] = mapped_to # llama-pth + # Attention key + mapped_to = "transformer.blocks."+str(i)+".attn_k" + tensor_map["model.layers."+str(i)+".self_attn.k_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention.wk"] = mapped_to # llama-pth + # Attention value + mapped_to = "transformer.blocks."+str(i)+".attn_v" + tensor_map["model.layers."+str(i)+".self_attn.v_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention.wv"] = mapped_to # llama-pth + # Attention output + mapped_to = "transformer.blocks."+str(i)+".attn_output" + tensor_map["gpt_neox.layers."+str(i)+".attention.dense"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".attn.c_proj"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".attn.out_proj"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".self_attention.dense"] = mapped_to # falcon + tensor_map["model.layers."+str(i)+".self_attn.o_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention.wo"] = mapped_to # llama-pth + # Feed-forward norm + mapped_to = "transformer.blocks."+str(i)+".ffn_norm" + tensor_map["gpt_neox.layers."+str(i)+".post_attention_layernorm"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".ln_2"] = mapped_to # gpt2 + tensor_map[" transformer.blocks."+str(i)+".norm_2"] = mapped_to # mpt + tensor_map["model.layers."+str(i)+".post_attention_layernorm"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".ffn_norm"] = mapped_to # llama-pth + # Feed-forward up + mapped_to = "transformer.blocks."+str(i)+".ffn_up" + tensor_map["gpt_neox.layers."+str(i)+".mlp.dense_h_to_4h"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".mlp.c_fc"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".ffn.up_proj"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".mlp.dense_h_to_4h"] = mapped_to # falcon + tensor_map["model.layers."+str(i)+".mlp.up_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".feed_forward.w3"] = mapped_to # llama-pth + # Feed-forward gate + mapped_to = "transformer.blocks."+str(i)+".ffn_gate" + tensor_map["model.layers."+str(i)+".mlp.gate_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".feed_forward.w1"] = mapped_to # llama-pth + # Feed-forward down + mapped_to = "transformer.blocks."+str(i)+".ffn_down" + tensor_map["gpt_neox.layers."+str(i)+".mlp.dense_4h_to_h"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".mlp.c_proj"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".ffn.down_proj"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".mlp.dense_4h_to_h"] = mapped_to # falcon + tensor_map["model.layers."+str(i)+".mlp.down_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".feed_forward.w2"] = mapped_to # llama-pth + + return tensor_map + From f4d137d98cf770d205cd788947cb990012593bd8 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Wed, 9 Aug 2023 00:50:11 +0200 Subject: [PATCH 082/242] convert-gptneox-h5-to-gguf.py : map tensor names --- convert-gptneox-h5-to-gguf.py | 36 ++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index 066ed0da07697..22508bd3dae56 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -1,6 +1,7 @@ # Quick and dirty HF gptneox--> gguf conversion import gguf +import gguf_tensor_map as tmap import os import sys import struct @@ -32,6 +33,7 @@ def bytes_to_unicode(): cs = [chr(n) for n in cs] return dict(zip(bs, cs)) + if len(sys.argv) < 3: print("Usage: convert-h5-to-ggml.py dir-model ftype\n") print(" ftype == 0 -> float32") @@ -74,16 +76,17 @@ def bytes_to_unicode(): gguf_writer = gguf.GGUFWriter.open(fname_out) -print("gguf: add metadata") +print("gguf: get model metadata") -llm_arch = "gptneox" +llm_arch = "gptneox" +block_count = hparams["num_hidden_layers"] gguf_writer.add_name(last_dir) gguf_writer.add_description("gguf test model") gguf_writer.add_architecture(llm_arch) gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) -gguf_writer.add_layer_count(llm_arch, hparams["num_hidden_layers"]) +gguf_writer.add_layer_count(llm_arch, block_count) gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"]) gguf_writer.add_rope_dimension_count(llm_arch, int( hparams["rotary_pct"]*(hparams["hidden_size"]//hparams["num_attention_heads"])) ) gguf_writer.add_head_count(llm_arch, hparams["num_attention_heads"]) @@ -92,7 +95,7 @@ def bytes_to_unicode(): # TOKENIZATION -print("gguf: add tokenizer") +print("gguf: get tokenizer metadata") tokens: List[str] = [] merges: List[str] = [] @@ -102,7 +105,7 @@ def bytes_to_unicode(): # gpt2 tokenizer gguf_writer.add_tokenizer_model("gpt2") - print("gguf: adding gpt2 tokenizer merges") + print("gguf: get gpt2 tokenizer merges") with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f: tokenizer_json = json.load(f) @@ -110,7 +113,7 @@ def bytes_to_unicode(): gguf_writer.add_token_merges(merges) - print("gguf: adding gpt2 tokenizer vocab") + print("gguf: get gpt2 tokenizer vocab") vocab_size = len( tokenizer_json["model"]["vocab"] ) @@ -141,7 +144,7 @@ def bytes_to_unicode(): gguf_writer.add_token_list(tokens) if "added_tokens" in tokenizer_json and Path(dir_model + "/tokenizer_config.json").is_file(): - print("gguf: adding special token ids") + print("gguf: get special token ids") with open(dir_model + "/tokenizer_config.json", "r", encoding="utf-8") as f: tokenizer_config = json.load(f) @@ -176,8 +179,10 @@ def bytes_to_unicode(): # TENSORS +tensor_map = tmap.get_tensor_map(block_count) + # tensor info -print("gguf: add gguf tensor info") +print("gguf: get tensor metadata") for name in list_vars.keys(): data = list_vars[name].squeeze().numpy() @@ -186,6 +191,15 @@ def bytes_to_unicode(): if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): continue + # map tensor names + if name.endswith(".weight") and name[:-7] in tensor_map: + name = tensor_map[name[:-7]] + ".weight" + elif name.endswith(".bias") and name[:-5] in tensor_map: + name = tensor_map[name[:-5]] + ".bias" + else: + print( "Can not map tensor '" + name + "'" ) + sys.exit() + n_dims = len(data.shape) # ftype == 0 -> float32, ftype == 1 -> float16 @@ -206,9 +220,9 @@ def bytes_to_unicode(): print("gguf: write header") gguf_writer.write_header_to_file() -print("gguf: write key-values") +print("gguf: write metadata") gguf_writer.write_kv_data_to_file() -print("gguf: write tensor info") +print("gguf: write tensor metadata") gguf_writer.write_ti_data_to_file() # tensor data @@ -242,5 +256,5 @@ def bytes_to_unicode(): gguf_writer.close() -print("gguf: conversion done, output file: " + fname_out) +print("gguf: model successfully exported to '" + fname_out + "'" ) print("") From 7d5f4522ddc2717e878966bf357dee790c33e860 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Wed, 9 Aug 2023 00:52:16 +0200 Subject: [PATCH 083/242] convert-llama-h5-to-gguf.py : map tensor names --- convert-llama-h5-to-gguf.py | 85 +++++++++++++------------------------ 1 file changed, 29 insertions(+), 56 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 67b3c55d4e35f..0b477a1336570 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -1,6 +1,8 @@ # Quick and dirty HF llama --> gguf conversion, GQA/70b wont work import gguf +import gguf_tensor_map as tmap +import os import sys import struct import json @@ -12,7 +14,6 @@ #NDArray = np.ndarray[Any, Any] - # compatible with python < 3.9 NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]' @@ -32,6 +33,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # output in the same directory as the model dir_model = sys.argv[1] fname_out = sys.argv[1] + "/ggml-model.bin" +last_dir = os.path.basename(os.path.normpath(dir_model)) # possible tensor data types @@ -48,6 +50,8 @@ def permute(weights: NDArray, n_head: int) -> NDArray: print("Invalid ftype: " + str(ftype)) sys.exit(1) fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" + +print("gguf: loading model "+last_dir) with open(dir_model + "/config.json", "r", encoding="utf-8") as f: hparams = json.load(f) @@ -62,32 +66,34 @@ def permute(weights: NDArray, n_head: int) -> NDArray: gguf_writer = gguf.GGUFWriter.open(fname_out) -print("gguf: add key-values, metadata") +print("gguf: get model metadata") -llm_arch = "llama" +llm_arch = "llama" +head_count = hparams["num_attention_heads"] +block_count = hparams["num_hidden_layers"] gguf_writer.add_name("llama2-7b") gguf_writer.add_description("gguf test model") gguf_writer.add_architecture(llm_arch) gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) -gguf_writer.add_layer_count(llm_arch, hparams["num_hidden_layers"]) +gguf_writer.add_layer_count(llm_arch, block_count) gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"]) gguf_writer.add_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"]) -gguf_writer.add_head_count(llm_arch, hparams["num_attention_heads"]) +gguf_writer.add_head_count(llm_arch, head_count) gguf_writer.add_layer_norm_rms_eps(llm_arch, hparams["rms_norm_eps"]) # TOKENIZATION -print("gguf: add key-values, tokenizer") +print("gguf: get tokenizer metadata") tokens: List[str] = [] scores: List[float] = [] if Path(dir_model + "/tokenizer.model").is_file(): # vocab type sentencepiece - print("gguf: adding sentencepiece tokenizer vocab") + print("gguf: get sentencepiece tokenizer vocab and scores") tokenizer = SentencePieceProcessor(dir_model + "/tokenizer.model") @@ -119,7 +125,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: tokenizer = json.load(f) if "added_tokens" in tokenizer and Path(dir_model + "/tokenizer_config.json").is_file(): - print("gguf: adding special token ids") + print("gguf: get special token ids") with open(dir_model + "/tokenizer_config.json", "r", encoding="utf-8") as f: tokenizer_config = json.load(f) @@ -154,8 +160,10 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # TENSORS +tensor_map = tmap.get_tensor_map(block_count) + # tensor info -print("gguf: add gguf tensor info") +print("gguf: get tensor metadata") for name in list_vars.keys(): data = list_vars[name].squeeze().numpy() @@ -166,45 +174,16 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # permute these if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): - data = permute(data, hparams["num_attention_heads"]) - - # chnage tensor name + data = permute(data,head_count) - if name == "model.embed_tokens.weight": - name = "tok_embeddings.weight" - elif name == "model.norm.weight": - name = "norm.weight" - elif name == "lm_head.weight": - name = "output.weight" + # map tensor names + if name.endswith(".weight") and name[:-7] in tensor_map: + name = tensor_map[name[:-7]] + ".weight" + elif name.endswith(".bias") and name[:-5] in tensor_map: + name = tensor_map[name[:-5]] + ".bias" else: - for i in range(80): # maximum number of layers - if name == "model.layers." + str(i) + ".input_layernorm.weight": - name = "layers." + str(i) + ".attention_norm.weight" - break - if name == "model.layers." + str(i) + ".self_attn.q_proj.weight": - name = "layers." + str(i) + ".attention.wq.weight" - break - if name == "model.layers." + str(i) + ".self_attn.k_proj.weight": - name = "layers." + str(i) + ".attention.wk.weight" - break - if name == "model.layers." + str(i) + ".self_attn.v_proj.weight": - name = "layers." + str(i) + ".attention.wv.weight" - break - if name == "model.layers." + str(i) + ".self_attn.o_proj.weight": - name = "layers." + str(i) + ".attention.wo.weight" - break - if name == "model.layers." + str(i) + ".post_attention_layernorm.weight": - name = "layers." + str(i) + ".ffn_norm.weight" - break - if name == "model.layers." + str(i) + ".mlp.gate_proj.weight": - name = "layers." + str(i) + ".feed_forward.w1.weight" - break - if name == "model.layers." + str(i) + ".mlp.down_proj.weight": - name = "layers." + str(i) + ".feed_forward.w2.weight" - break - if name == "model.layers." + str(i) + ".mlp.up_proj.weight": - name = "layers." + str(i) + ".feed_forward.w3.weight" - break + print( "Can not map tensor '" + name + "'" ) + sys.exit() n_dims = len(data.shape) @@ -227,9 +206,9 @@ def permute(weights: NDArray, n_head: int) -> NDArray: print("gguf: write header") gguf_writer.write_header_to_file() -print("gguf: write key-values") +print("gguf: write metadata") gguf_writer.write_kv_data_to_file() -print("gguf: write tensor info") +print("gguf: write tensor metadata") gguf_writer.write_ti_data_to_file() # tensor data @@ -237,17 +216,14 @@ def permute(weights: NDArray, n_head: int) -> NDArray: for name in list_vars.keys(): data = list_vars[name].squeeze().numpy() -# print("Process tensor: " + name + " with shape: ", data.shape) # we don't need these if name.endswith(".rotary_emb.inv_freq"): -# print(" Skip tensor: " + name) continue # permute these if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): -# print(" Permute tensor: " + name) - data = permute(data, hparams["num_attention_heads"]) + data = permute(data, head_count) n_dims = len(data.shape) @@ -255,16 +231,13 @@ def permute(weights: NDArray, n_head: int) -> NDArray: ftype_cur = 0 if ftype != 0: if name.endswith(".weight") and n_dims == 2: -# print(" Converting to float16") data = data.astype(np.float16) ftype_cur = 1 else: -# print(" Converting to float32") data = data.astype(np.float32) ftype_cur = 0 else: if data.dtype != np.float32: -# print(" Converting to float32") data = data.astype(np.float32) ftype_cur = 0 @@ -273,5 +246,5 @@ def permute(weights: NDArray, n_head: int) -> NDArray: gguf_writer.close() -print("gguf: conversion done, output file: " + fname_out) +print("gguf: model successfully exported to '" + fname_out + "'" ) print("") From 0246d0dd6f8f57b64b696a33362ecf287d33ef63 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Wed, 9 Aug 2023 00:54:21 +0200 Subject: [PATCH 084/242] gptneox-main.cpp : map tensor names --- gptneox-main.cpp | 66 +++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/gptneox-main.cpp b/gptneox-main.cpp index eecd59678a3eb..1667c4d545114 100644 --- a/gptneox-main.cpp +++ b/gptneox-main.cpp @@ -549,56 +549,58 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 model.layers.resize(n_layer); - model.wte = ggml_get_tensor(ctx, "gpt_neox.embed_in.weight"); - model.ln_f_g = ggml_get_tensor(ctx, "gpt_neox.final_layer_norm.weight"); - model.ln_f_b = ggml_get_tensor(ctx, "gpt_neox.final_layer_norm.bias"); - model.lmh_g = ggml_get_tensor(ctx, "embed_out.weight"); + model.wte = ggml_get_tensor(ctx, "transformer.token_embd.weight"); + model.ln_f_g = ggml_get_tensor(ctx, "transformer.output_norm.weight"); + model.ln_f_b = ggml_get_tensor(ctx, "transformer.output_norm.bias"); + model.lmh_g = ggml_get_tensor(ctx, "transformer.output.weight"); // map by name - model.tensors["gpt_neox.embed_in.weight"] = model.wte; - model.tensors["gpt_neox.final_layer_norm.weight"] = model.ln_f_g; - model.tensors["gpt_neox.final_layer_norm.bias"] = model.ln_f_b; - model.tensors["embed_out.weight"] = model.lmh_g; + model.tensors["transformer.token_embd.weight"] = model.wte; + model.tensors["transformer.output_norm.weight"] = model.ln_f_g; + model.tensors["transformer.output_norm.bias"] = model.ln_f_b; + model.tensors["transformer.output.weight"] = model.lmh_g; for (int i = 0; i < n_layer; ++i) { auto & layer = model.layers[i]; - layer.ln_1_g = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".input_layernorm.weight" ); - layer.ln_1_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".input_layernorm.bias" ); + std::string blocknamestart = "transformer.blocks." + std::to_string(i) + "."; - layer.c_attn_attn_w = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".attention.query_key_value.weight" ); - layer.c_attn_attn_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".attention.query_key_value.bias" ); + layer.ln_1_g = get_tensor_ex(ctx, blocknamestart + "attn_norm_1.weight" ); + layer.ln_1_b = get_tensor_ex(ctx, blocknamestart + "attn_norm_1.bias" ); - layer.c_attn_proj_w = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".attention.dense.weight" ); - layer.c_attn_proj_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".attention.dense.bias" ); + layer.c_attn_attn_w = get_tensor_ex(ctx, blocknamestart + "attn_qkv.weight" ); + layer.c_attn_attn_b = get_tensor_ex(ctx ,blocknamestart + "attn_qkv.bias" ); - layer.ln_2_g = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".post_attention_layernorm.weight" ); - layer.ln_2_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".post_attention_layernorm.bias"); + layer.c_attn_proj_w = get_tensor_ex(ctx, blocknamestart + "attn_output.weight" ); + layer.c_attn_proj_b = get_tensor_ex(ctx, blocknamestart + "attn_output.bias" ); - layer.c_mlp_fc_w = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".mlp.dense_h_to_4h.weight" ); - layer.c_mlp_fc_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".mlp.dense_h_to_4h.bias" ); + layer.ln_2_g = get_tensor_ex(ctx, blocknamestart + "ffn_norm.weight" ); + layer.ln_2_b = get_tensor_ex(ctx, blocknamestart + "ffn_norm.bias"); - layer.c_mlp_proj_w = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".mlp.dense_4h_to_h.weight" ); - layer.c_mlp_proj_b = get_tensor_ex(ctx, "gpt_neox.layers." + std::to_string(i) + ".mlp.dense_4h_to_h.bias" ); + layer.c_mlp_fc_w = get_tensor_ex(ctx, blocknamestart + "ffn_up.weight" ); + layer.c_mlp_fc_b = get_tensor_ex(ctx, blocknamestart + "ffn_up.bias" ); + + layer.c_mlp_proj_w = get_tensor_ex(ctx, blocknamestart + "ffn_down.weight" ); + layer.c_mlp_proj_b = get_tensor_ex(ctx, blocknamestart + "ffn_down.bias" ); // map by name - model.tensors["gpt_neox.layers." + std::to_string(i) + ".input_layernorm.weight"] = layer.ln_1_g; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".input_layernorm.bias"] = layer.ln_1_b; + model.tensors[blocknamestart + "attn_norm_1.weight"] = layer.ln_1_g; + model.tensors[blocknamestart + "attn_norm_1.bias"] = layer.ln_1_b; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".attention.query_key_value.weight"] = layer.c_attn_attn_w; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".attention.query_key_value.bias"] = layer.c_attn_attn_b; + model.tensors[blocknamestart + "attn_qkv.weight"] = layer.c_attn_attn_w; + model.tensors[blocknamestart + "attn_qkv.bias"] = layer.c_attn_attn_b; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".attention.dense.weight"] = layer.c_attn_proj_w; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".attention.dense.bias"] = layer.c_attn_proj_b; + model.tensors[blocknamestart + "attn_output.weight"] = layer.c_attn_proj_w; + model.tensors[blocknamestart + "attn_output.bias"] = layer.c_attn_proj_b; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".post_attention_layernorm.weight"] = layer.ln_2_g; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".post_attention_layernorm.bias"] = layer.ln_2_b; + model.tensors[blocknamestart + "ffn_norm.weight"] = layer.ln_2_g; + model.tensors[blocknamestart + "ffn_norm.bias"] = layer.ln_2_b; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".mlp.dense_h_to_4h.weight"] = layer.c_mlp_fc_w; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".mlp.dense_h_to_4h.bias"] = layer.c_mlp_fc_b; + model.tensors[blocknamestart + "ffn_up.weight"] = layer.c_mlp_fc_w; + model.tensors[blocknamestart + "ffn_up.bias"] = layer.c_mlp_fc_b; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".mlp.dense_4h_to_h.weight"] = layer.c_mlp_proj_w; - model.tensors["gpt_neox.layers." + std::to_string(i) + ".mlp.dense_4h_to_h.bias"] = layer.c_mlp_proj_b; + model.tensors[blocknamestart + "ffn_down.weight"] = layer.c_mlp_proj_w; + model.tensors[blocknamestart + "ffn_down.bias"] = layer.c_mlp_proj_b; } } From 1c4d8bf98145e2a4c0955e154d74aa579c6d19ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Thu, 10 Aug 2023 16:52:08 +0300 Subject: [PATCH 085/242] gguf : start implementing libllama in GGUF (WIP) --- Makefile | 8 +- examples/gguf/gguf-llama-simple.cpp | 182 ++ gguf-llama-simple | Bin 0 -> 607488 bytes gguf-llama.cpp | 4060 +++++++++++++++++++++++++++ gguf-llama.h | 468 +++ 5 files changed, 4717 insertions(+), 1 deletion(-) create mode 100644 examples/gguf/gguf-llama-simple.cpp create mode 100644 gguf-llama-simple create mode 100644 gguf-llama.cpp create mode 100644 gguf-llama.h diff --git a/Makefile b/Makefile index a3600e4f215a5..f5922c95d2f98 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Define the default target now so that it is always the first target -BUILD_TARGETS = main quantize quantize-stats perplexity embedding vdot train-text-from-scratch simple server embd-input-test gguf gptneox-main +BUILD_TARGETS = main quantize quantize-stats perplexity embedding vdot train-text-from-scratch simple server embd-input-test gguf gguf-llama-simple gptneox-main # Binaries only useful for tests TEST_TARGETS = tests/test-double-float tests/test-grad0 tests/test-opt tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0 @@ -337,6 +337,9 @@ OBJS += ggml-alloc.o llama.o: llama.cpp ggml.h ggml-alloc.h ggml-cuda.h ggml-metal.h llama.h llama-util.h $(CXX) $(CXXFLAGS) -c $< -o $@ +gguf-llama.o: gguf-llama.cpp ggml.h ggml-alloc.h ggml-cuda.h ggml-metal.h gguf-llama.h gguf-util.h + $(CXX) $(CXXFLAGS) -c $< -o $@ + common.o: examples/common.cpp examples/common.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -393,6 +396,9 @@ embd-input-test: $(LIB_PRE)embdinput$(DSO_EXT) examples/embd-input/embd-input-te gguf: examples/gguf/gguf.cpp build-info.h ggml.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) +gguf-llama-simple: examples/gguf/gguf-llama-simple.cpp build-info.h ggml.o gguf-llama.o common.o $(OBJS) + $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + gptneox-main: gptneox-main.cpp ggml.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) diff --git a/examples/gguf/gguf-llama-simple.cpp b/examples/gguf/gguf-llama-simple.cpp new file mode 100644 index 0000000000000..35c3c818349be --- /dev/null +++ b/examples/gguf/gguf-llama-simple.cpp @@ -0,0 +1,182 @@ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "common.h" +#include "gguf-llama.h" +#include "build-info.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) +#include +#include +#elif defined (_WIN32) +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#include +#endif + + + +int main(int argc, char ** argv) +{ + gpt_params params; + + //--------------------------------- + // Print help : + //--------------------------------- + + if ( argc == 1 || argv[1][0] == '-' ) + { + printf( "usage: %s MODEL_PATH [PROMPT]\n" , argv[0] ); + return 1 ; + } + + //--------------------------------- + // Load parameters : + //--------------------------------- + + if ( argc >= 2 ) + { + params.model = argv[1]; + } + + if ( argc >= 3 ) + { + params.prompt = argv[2]; + } + + if ( params.prompt.empty() ) + { + params.prompt = "Hello my name is"; + } + + //--------------------------------- + // Init LLM : + //--------------------------------- + + llama_backend_init(params.numa); + + llama_context_params ctx_params = llama_context_default_params(); + + llama_model * model = llama_load_model_from_file(params.model.c_str(), ctx_params); + + if ( model == NULL ) + { + fprintf( stderr , "%s: error: unable to load model\n" , __func__ ); + return 1; + } + + llama_context * ctx = llama_new_context_with_model(model, ctx_params); + + //--------------------------------- + // Tokenize the prompt : + //--------------------------------- + + std::vector tokens_list; + tokens_list = ::llama_tokenize( ctx , params.prompt , true ); + + const int max_context_size = llama_n_ctx( ctx ); + const int max_tokens_list_size = max_context_size - 4 ; + + if ( (int)tokens_list.size() > max_tokens_list_size ) + { + fprintf( stderr , "%s: error: prompt too long (%d tokens, max %d)\n" , + __func__ , (int)tokens_list.size() , max_tokens_list_size ); + return 1; + } + + fprintf( stderr, "\n\n" ); + + // Print the tokens from the prompt : + + for( auto id : tokens_list ) + { + printf( "%s" , llama_token_to_str( ctx , id ) ); + } + + fflush(stdout); + + + //--------------------------------- + // Main prediction loop : + //--------------------------------- + + // The LLM keeps a contextual cache memory of previous token evaluation. + // Usually, once this cache is full, it is required to recompute a compressed context based on previous + // tokens (see "infinite text generation via context swapping" in the main example), but in this minimalist + // example, we will just stop the loop once this cache is full or once an end of stream is detected. + + while ( llama_get_kv_cache_token_count( ctx ) < max_context_size ) + { + //--------------------------------- + // Evaluate the tokens : + //--------------------------------- + + if ( llama_eval( ctx , tokens_list.data() , int(tokens_list.size()) , llama_get_kv_cache_token_count( ctx ) , params.n_threads ) ) + { + fprintf( stderr, "%s : failed to eval\n" , __func__ ); + return 1; + } + + tokens_list.clear(); + + //--------------------------------- + // Select the best prediction : + //--------------------------------- + + llama_token new_token_id = 0; + + auto logits = llama_get_logits( ctx ); + auto n_vocab = llama_n_vocab( ctx ); // the size of the LLM vocabulary (in tokens) + + std::vector candidates; + candidates.reserve( n_vocab ); + + for( llama_token token_id = 0 ; token_id < n_vocab ; token_id++ ) + { + candidates.emplace_back( llama_token_data{ token_id , logits[ token_id ] , 0.0f } ); + } + + llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; + + // Select it using the "Greedy sampling" method : + new_token_id = llama_sample_token_greedy( ctx , &candidates_p ); + + + // is it an end of stream ? + if ( new_token_id == llama_token_eos() ) + { + fprintf(stderr, " [end of text]\n"); + break; + } + + // Print the new token : + printf( "%s" , llama_token_to_str( ctx , new_token_id ) ); + fflush( stdout ); + + // Push this new token for next evaluation : + tokens_list.push_back( new_token_id ); + + } // wend of main loop + + llama_free( ctx ); + llama_free_model( model ); + + llama_backend_free(); + + return 0; +} + +// EOF diff --git a/gguf-llama-simple b/gguf-llama-simple new file mode 100644 index 0000000000000000000000000000000000000000..d7600282e3315a4bc805a4f6c01550d58bdb80a5 GIT binary patch literal 607488 zcmeFa3tUxI7C(N#i=t`w`YL+giW;0=;1e?IA`j2ii;|LQTA&C*ArM>?O%uCNo^E%d zvKectvB%8xm>Hd>2lOyd5wsUtQ&v-wf8wJPxGWlMnow3PcP+ag($~Nj=01H*SNkSrzkEZQ8@*_&r{A- zdIPoLze`*-rv;Jxf}TbTzFtV_zNf`?b9%Pm(^I{Ml+t}xKb;f!6+KlPoTmGtD4WS? zb9&L2)DlI}Q~5sjJXarnqbcn_h}TR{<@=~a$925Lj=b(^j-3}5UD%&zL_J9ybYJ|9 zoIid+_tYumrKkKVmkHH#lelh9ow!CnPWC%2!^|(e^XR_lfAIF1)1pEA20hhfe=Y9g z5)%cUAQ8<|ri$lT$)%$oFGx?hbQR@_qQbeOM_y4h@3Nx8;)=@4DicRvHhSdc{dn)H!RJSMGZc(3oP)tw!Pg}_6EsN(S!4P^X$A)(GB)3Ht?Hvtg z(Nj6Ez_5D4>n+NeZBhSgEy_=YetRnaJPZ3-Y$4|lE%*;u;4TaKKWBkwqo4Lv&r*x_ z_O;OadJFzHEbO__LhnCW$lnh+_oVj^E$WkHVL#Vd)bmUW{7sAcJZ@n>KUm-!2J|fF z&n@s_7W$2~z}Hy#f#<+~K@a}tVT<-UEXsFT)U&TeIk^_N(?Tz!E$m^ZMLFMC;8QH@ z;c*N8wHE%cw?#cQi++8X1^@jPa%-}v&ms%_Hj94RYJqRH(Bn1>{1S`jeq@0^41dy7 zf4s$lzn6vFR$1`xwD22mSm^g`i+;DoLO!+V*FDw0z``DE7IN#bsOL=<{&t^5eb!mX zA>IOCW`Vy9yr*`(WFh}YEc#uY#kdw>@!a7S_J67cKE|S)D=g&ZwBY~F0&lbMhXXC} z6pM0JL+?HL&-X0&&$B51MGL$7(Za3zUdmifJ#DDY8wHEcc z+Ct7lE%@KIsLyzd`UEWG-_N4_Wfp#HwuK#zx6tF67Uh2qKhu+6`pSYI#rG)Bu@?AS z7V_+{cjO=-NWqI=p%e{GJSs5usCB=DJbLJN1 z@oPO^DfJeV<;|Iwor~XIFlOt(EqRM|P={_VDJ)j9Z z5!vpN^6V+4dBqD#=Aj-pOrCU{JCAu7FKA3|$%4{3WqE0tlX7`QM;Ddk&MC@EO-Ne^ zV48P$e0DZkT(T&;D6e?Fw;(&OtgNJrvrqGuq24_-WGdB9G+4AZD?2MbJ8jzN?5Wd6 zK=e0cO`qmX%B#%HEAoy>jKU+hy>NgeA74>icxy!-6kq0^JS}^4_B1anfv$|4mzO`MqR0!YD$4WbxuLMM zw9NE{3)7e^lVD{F3X8!^vPD%#^FoKm!w%*xf*!JSONw(V%F6PJa~CrOOrBO)R-QJs zkn<-NLgkQaUg~gmHOp-hVpbN_;v`Isl~*{bJTLE-`KbBG{L+$gcS1sT_5$=6R5061 zRVKq7QC3mxEnJYt6@Nk#wGc8&gH6xLo?n5MX3xpJwW6?$8SS(Z(qdt8d0v^a>!q|4@-B!%2a#1ZH5eB_zuI$hrgaK7l3*(@)BtIv3qDFHh7jdrtAZ?6SPMbBg8^ z=jP4Lgme-OmSXfmrkiQ;uyN6ZlhOqqhiC?sKcHW@fZvHZd!G&*feVR?2594GY`*1Oq2!9S*~Ty04gvfw?$4caAr=fRz~@ipm#bz+S*T z61hh5gC>jZ?r-LnlzG7?3*sboKe{Zhyl@HBPUGXeqD<}@{-4l1YjeVAS&q4L%DuX* zxhE~dK!fpsw@nNuVn7o8?muXAVrfY!^p<;zp;)Fs8WyAxJ9~a*Wp-&^S$Rn@9Fuo( z_QH5nBMU>KKCbB_Bd?uFLE-Kwsy&isOU(8D2UVUkEj)!WJmAl=buqGKA2aD1o{a&; zV#+nGkoktZN=)K%#rRb?l?`C9Na(JFk2d-McY?{1;C366rA!*k$g$6x!z_>Hc%yQu zr=a^`yvWF2R5&lM7{wQQx#+-Owb4|5?`B!~9%sfEcX*gHd#(c@Whn^5pX^#+tOSs|io(c@3GsF}0T5 zxHM;)mj~sr$qaf+4^t_~9~Y8z7g$s>AA?VL0m2H;r}(0GMv_-a+_#vZ zied(${9N9a{QRPd@&YBFMq5;`ys$ERLEZwm-TB4J0>O>?7ZoY+H^sT7iyU$78m9f%~SF*-6>bFI>gPy zgax7KQ5YF23%v@egqAA#w5m`RutzK`FD}L6m4xq|2Mbm5sZPX6mCQ#yFnF;CR^W+x z(O8t1QM;%HWOv0SO8%m)jXyiSfk}D6H-#L zM?mKjJnr!+*~2d%o)pfG31=@KW#mU(K9c8W%)p%^FCQ*$7G#3BOD7Y%Gmq{@CM0wt zla!PhGm^)ksUH)9~SP89a}M==DH0+>V5i6mf}Edf{9`B>qJq*IVg> z-}J9H`$tzJk>hkU%SS7H#T|t9WA_qYFD07&r?Mh(?xvs8pV586-y3%fK-x<=m0gWe z>`19>`F|wtIfY%P68MvO$-MO5IJ81_8EEkIMvSPpLGjY59+9kugGCt*&{T&=We9## z8jXK8ly(ZMGeTU-P@t!=tJFSfStPDg9ds!XML1z3Ed&(t6M=szh_&hQyv&OiBa%5( znN0w@-b*=vm>ZEU{MYXLEgpB8G7kV@cQM+blnA=w%tAzR`zlN5B7P$9tQ=8(JZSG3 zbbW|&XE%PE#D5Cz;bk`}9mhV(K|$Zj?uXP6L_*pjRYWg68szG~}V5XTn3+Plq@! z79-9%>G--C{JPVGw+TGXgs&6js3ts~4v64qh6!)Y6OvpD{=33r{t@dgu~`2u-0MQJhN@h@|{)r7Bk zgyTC*xMMIj2JjO!;cZQvzr%!QZsB;R2`?4TRpxisd(Kc^zQcq&TX{J#CcNq+jyp|w zt$1#n32%tuUmfftzYr2;QC;VT4Q zX~ORmc$EqF3w)&suN8Q$34c)Ft4;XB0$*do{~++EP54s+UuVLf5qO;m|BJvIO!zv1 zx0>+9)4BY2nD7?_f13%f6L`>szb5bw6W%EBP7~fFaHX)jooo?!vxN5==2|Uw;9~Jlv6W;Geu9qAW9wYDq z6Fx-Xr6&9wfmfRFO9WnJ!bb>vr3t@E;I$@vyueqR@N|K%G2s&h{HZ!qEW1m0r8iv`|l!WRjAhY7C{c$*2oTi`(xzFOcNCj8F=?=<1-1g_lN-Tv1L zJlceB6u85Lzb)_>6aJpSohJMvfybHfodSwhfe9Za@KO_=An-~PK33pWCOlo>D^2)Rf!CVwn*_evgx@UiH70zS zz#B~Xg92|c;V%fh)r3DR@EsO-n+bnR@CQx!O9JmO;jak1(}b@VxN=K(yWJr0XcN9! z;0_c1rodxN_;!IiP51`_k2B%_5_r4`-zD%w6aKBhT_*gXz*Q3-l`ZVngr6qx876#? zz;jIaxdJaR;a3Q})B>+G;Uffpl?fjs@RcS!Mc}n2+%53cCj44~uQB0c1^%=NpDplp zCValY>rA*;;0-3cO5iOf{62xVn(&7NzQcq+EATcG{+hspCcH)99VYxEfp?nlodQ>i zy4(L=fk&J0Ljrf0a3zP^Rg4KgMc_^oewx7JO!)Z%k2m3$3Ovz-Uny{x2~QWeYQkp< zJkx~d34DeLFBf=@2~QSyfeBwK@KO`*6L_Tw4+y-qI#Y6W%ItrwLyt%Fpz6w?mz7 z5YMI6JiVJ`{D^YQxZZBUr{h1x#OOz_uNeK2_MWM z_B%(y=^Zour$EBb&_Sf768@NkS4ud&r>6f@NqC$NB3&urDH2{Q;gcnNwS;F&_!vmGB&)chZZD7g*p8QaSfa<+Mon0}|dU;SWmq4hjFAgttlf zY6%ZY_(KxjA>j{8_zbE3%OpIel-m#WDr&R-3wtlr<7QRKlN@@Y5uGorIq*;dK%oBjF7a zF8i?-2|rWfZ+eL<nlPT>t5maHkGZdTZEj&zJCM z3BN$X9TI+_gvUsDtb{uy{2~dDlkkfrJYK?oCEumS0(%s3D1=9OC@}U zgkL7%ITC)kgcnHo6%t-5;qek)Dd7ncUM1nfC48lXkC5H!s{e_wuCoG_>B_YBH=elc&mixNcav3 zpCjRI5(&6D53sgu5iXSi)5aFOl#}2``oK84`Z0gy%?jnS>Wec)5g^O1M|TDC*k)>c)Wz)C*g?_zDmMf68>8W|6lfh68N74{wIO|N#K7H_@4y+|0{v- z?dKj*ecwi_{;0p77pQVWRsIoyq9?iS9?Vi_v!zeG1X>jJ|{D{zN+&y@coiL^~K=Ml^*cLkgpB zCfZJP$58+m%_W*bk)bw5-$3+0qFWg~m1qh*h8h?>k?6rhuVeIhqK6Q@hS66NO`*q7 zEu)7MO(Dln6{CMmG=&;NrHsCSXbLfgau|Ik(Pt2y$>>2uQ%Eu7Vst;EDU=wBXLKad z6haI+8GYnj&=fihIT*d4XbKsI6h?nTG=&O79Y0a~cMt&HAI^aVsW zFnTM|7ZSaW(Hn@S5MXExqhBHVBBE;<{XEeZ6J5pVXNmq5(WQ)jg6KG+a~SNLUExgM*o^< z3c-a+8GQlK6nYEgF#1fQDdZN)Wb`1SDbyBnF}feo6k-d-GdhxJ3ay2lj6QM>XbP!? z9E{#iG=y`N|bNreCYLYa&nL^OqRLM}%4Bbq`u zp?F3|5>26-kdx6z&H}xRXa}SB6MZ|;3ZuUvnnF6EjzdiUL{|~r#^{fUUO{v#qqh@% z2hk0T-b%EO=yiJJ;|^8(JlF>R ztop|dI~`t!=wW z)pn`A-5r@(ovjt_9_G4J|`42Wr*8xM5UKAnF22k~+OZF_oH3kHA!_ zZ&#cByeg{4Mv;R5@R`IvYCk2zj;dOR+PL=`wXw688nId3cGNor1zaWyi0*1bQ{XN2 z`>B3a<6gwQO2sADslIU!4@FkhzV!BoC~gNHtjDk&oCj5HjtYQ|xDoeuyFauD7e#%p zOTIpNdh&J2S<}=N-^TQ!qQ2LDQUkBW22tx^1}0*@pCY_{ReyPO)zz0(j8Xj~(1we? z>a8R+1kXL#)%AJ=7>ygw?u$5CP+hT!%9@otJNd@so079{-01r$a_NP{RR{Lq#qg}Y zpL&%IM6nJ+I|(KKs|Frngz9@O7O!Zbm3GMLpZGd6X?!nKWq;|l*uf|~IOld7d+r*l z=x%(7O7-*NReyDC8$eY{jdeBs=%|h*7G&;;ZD5&!o>*toj{|M7O`VH^H@{_5+}dWS zt0ESX_YHws{E2wzt@O~qP(KnhwHGSKW{4+Wx<@-i519EaJ)p^>y%1Z4$E$&*u>~p~ zsC`^dq6>a>NpHpD2k7z7i!D|Csj-z*^GuKST`*%>l#&{7{pitldWIcV8$*$*e`wp_ zXxz{*IjLpY1*qqDyIDQIK;qHfMO9}&N=z%EDyGot*p=+L+vIJucj|&p#+b%G*%71enk{G?_m{4YOx32;}?T}1fDVMM|b15k*Uzu zx2O}l=V+?7OKS`s92ce3G}voC5bWu%kO((X#m+)jYr@5qAzZXqS3rhp?x$+N_9zO0 zbgyy+PlqE>wMJmG)WF#*w7qEwU2g4dkM>mTYT|(=YLT|b=GUfbNlixvCfk~m(T%7Q zdsJUze|J~2C+VoY`fL=wX~hzU?KL&&ql))D!`jt4a*Fs_t|o1?SI{d#^%e zQe)>$4A}li)nTo1qm!vxQ_~R#uC$-6YTLTqSk~;;-om{Vn-ca+35;Brt}T60+4T65 zfvGRr#y6+F$l9gWd~*A#Drsde>@GMRBU?tz9&a>Bbj!cHx+VrZFDjD)ws~q`b`4sz z$Kw~W@^e{9QmOd{4yN6fuqR=Q3gs85!$NA~VX6J9eQk@Xd16a9!<3=^rLi;AfS2r> zOycfFo06<;4&MJaN}H0CHruQ3rM41Zs&+7_;R0ODQy2AtrpWy2_CWytsK*`L>grKn z>Lr=(q-O8HG;K^qV7^VQ*;8>%Xdyg2eK{B{Xbuh}K5b*F77x>L`!?C=37Kk5mp3Z- zPgt}^n};&A&vC20`YIY5AdJ^=A#?>BD55Ia1Ec8^#k|L$R$K9YQ*aE4Py3Oc`DQ2* z%p}ICd$%XYlr*2j`$rc$)K( zpt%o0-=V9}G4Z%Z=y8erS-y($mm<%)R{NRlG3}?d4{0C7$y0b@YhT#zC3oh7tQGG? z?JC28wD)hH)jqQwqS;JVj=G=2nJO+K56s;CP4E}!^}#=}4)k5{Z{Op|yrfS^dTQ_y zLUi^k1p6Ye(=&Dabb)6B&%azBirF~EhMTNunX2|fb06wQ!JEI2QnVCXQ)gV%^3QuK zO`QW%qSZPGkuASfEhVO@}5HC8-hsap5;TlZY!0<3Djd)U>%{#BIXDHnmO+4l%2 z`oZ&jYmj;ZPg2D^+LKQsQJa11kkHR&AFFV)uZ59Rt5;i*RRg{@N;Ds%nr*Q$jQlEL zd$5dkYktAsKUUTJ#M;#9h+4?2sHm^tJ6T1S&n+xgwM+*XcF@g!cFSfPIzzxmXUR}{ z2QD--qN4bg;$~kjz{Mqtic!T*T%zd7W?vr?5AJdDdmNNB`(g;i&t`_xbr-)L&#xyM z*9oWVIsAGizdpmbPB>k!;@3<0^-AM9;dFfszh29)uQsj|j`m;Q0JZtn@v9BuDsJ0B zh~{hI7h1)I%?zhvJNfk>zusY7CmgMdhNq_XI*`(%^U-vIaEV?GjcE2=z(~dK{ zCt%N=&2iR$0<72Y`lJF!Pt8>Qw=oxex|pAO{MQ8I=$eKBtE-{|*IlZAnyNPL>XmRX zn6am;i%e!yW97i!?3!<5shaeISDi&V3GLYn4}>4X3Aufn9gv=Pu-kt%!(nBLM9yng%K?aPMUKV$_XAzVhG7ol zhosZ*)O05$^U__Ec+zv+z6O^kX+uSgn$+lx!>~6DH8JFyYJb@_1+y22?G8+ZY`;;v z8r7thiXbM^SjM=Nge}yvX!||P7Y5W$x3$qr7rIOXJ24yWRYs=l`h<#E8pfi(t@$$%{W&ouSeWT0mYwWbwRASi6!0Ipt@&}tyA8snj4}$NmwvsK(u5khCQKgbdw+UI$~%t8vO=bl@k2xH*k-2pNxFz z!#h4ZD@8q26zfn6J+aY2ws2}y8-rMx9~q=Zw5ky;aP~1LDDOsA-n)mUeeuWNZ*qpq zbB(uS@Y2icV_jb8ln{9=!(Y`=O*V$?68Xke z&W*-Gt0oac9)c+??EV*G8z4Rpma_5Gy~6cdPeT9(B&;$}^50N$SIamuO1p0c3Sd(A zXa_yok>Con6pQY`{sE`s30;la06S9O;pnng`~#wqU2(Vzg)Pd%Q&*dxs@rq-=Upv7 zkA`>oc?0o0|A4h~qOr8BaM1IfM1I9#3Nw_w?2o#SU9nexfZEnK4?M&8!km&4oNupF!xrJ9ihiKcY z$^IIAIz|bo5y@JBa>-gzjne*PbJR0ApzzZ^3-hIg)3L(1V;7mls2hLPM+uhTehOP% zjb)RneZ8J`QB-Y1@MW@U;OExG_fwP+wpja1p4iCzzKsYv1;)J?iCgCp3nDc4e2sL) zCbAqU`U69}q!$@tH27y@9kYHIie^!WW0*r9p^3QRNThDLoo&o=`{F)W?tCQmTd^jC zG>W0p;LI->|5k&)h4X)e1pH%&pZ-h&_^&4ZVVwVAgMT&We}?n-A^tkfe=6}G-Yvxw zed7wV@f{@l0}U9C5SQ~Wic$TitNtm`9{&V{vc_G9+Y=f>wlAnVxfIOyu+}-^A-=%4 zN?e4n1B@rwwu{DrgWk(vUgrX0{QiY8ZvSa+|K#Wle>x`ppQG9@6Qh5j1I!hlp%}hi z9F@+B;Zx9?Lid9yzgG26h{5_BRlzSO{fZ-(W%v_ZXDDNe+erLzsJ%*ScC;ff>XuJY zMZEyVMf|o8aH)eX<&OLFtxST|IOV!GP_D4q`l4?GrHGG}u*NTL+hq zn!A~m=0_NBsGXN>+lu^?tg!xfv%{Ma%pGDyHWA+38#OUUK zp(1`;LVbx`V}Vf_RMYM%>d#aJ~xl~;)B<50(@J1U&W#UK|`aT;{33sEcL~fmgo$+}e(=0cRi|+=NbQTewN0&BF9$Bj#KvsWUE>o-{M+ z>vs{=`HJ1{81MlI6w}zW|0b-0<0wpx(rHS*>5D>R&cDfrGA>r|1hgyoEPwQXhv{BX z>`mMB$8Tlpge?l))BkcMxRki}eMf7ENPP{#>U23quV$=>A}P@8UI8nQz@YyrRzQ-V zZ{OLCEMerhM0nCy;%2~-4+dZRycVgo*>rl>29(4bx658DLGZ`7&9ZPzC4H_KNXv7vEfu zWv#XY!ywlD`hJs`{l0XB9DO35_QDQ&=+Gu4@V)Mj;d|o;(iQ+bbgjYl8_x9y&NY^} zhJz~sopDhx0=69d_FpWZVE9LhGyTIk<+m>OQP~$<+N9)U*z;AonFi7AvDb+{%1HnPp6;c^b0!uD5q@{$0YvWaXL$< z@8k4SI_>B50iCYlbSkZH=>FxLUZvBEIlWt_%Q-!=x8PsE>1v&x&*}GddJd;!`w0G7 zoc8MUb)4R$(^EJ-BwFyhIXzFOQ#k#CPLJcXjaC;_|1q4-(&+?FKc&-`aQc8wU&QIu ze&YV~IK4`z&*1cKou(~Q*w@HY1pfd|SL<{iPQS0y5uA?gFZh3eW5NAio&GncH|g|u zoE|bj@PEbWc{=?mr(e+N_c?7lRq(&b=`5Yz!0D%Sx}MVqbb1}9Q|;pZKXZDOPCv%! z-8%gsr$^GJ3(1@Qa4FhTte5j71{GoQr(HbwblXBRNS3)0B4huIW^fu*Aq1;Bw^`{(dLe~wT+*-<=O1VE$ zj<&r*zo#5I|ImGugELe@cTsK-<(5%yFy+cAH-vJ9lp9L9Y|5QRx$7v$&M;1*90pJ& zluWrZC^v?3XHxDm%AG~I3n_Ot<<6koIh3Z&2=H%DqUrUs3Kk%EeLcamvv$HS{~m(ONhZpxh;t^HJ_n z{AyoXcNrxWb(d38Tz3T}^XuX%$*D`AWM)cMo}`ZZZsvM>&8%W zMO`8#adk+Z+m= z*SIY@xOPWZ7Y3k~%puAw3W=rS2LBYNTC>HA1HZ}9YOe@3+n2^GHINmHMf#LD)weuR z@kXisi1x^Y2Gw7Nwdk!5Y)?gEdnUqIpV?oUFc51jV?1=NT-1%m3Lsj(uKN3!)}xDJ zu#b$8d-WY?560$i-k~9SfDhBz_KRv0w!05~?@rp_wollguPoU@GrdK{iWoO~Gkmwl zC>8s#I>8$J+=>3wGvg-vv(8kp4c|J^zBctt2m2kvemlqehn(rgg0?jy_w4(y`1JJs zm=>QEzfIVleCYf3RjSq^*B8O^_EV(xvG{+owk=uvoZeeDw|{!H zyVq$f=w7vapn|`o0)`2R#JdPJjPv-fk4N;+lhkUzle@@qA0o1Z8Ok|0oPlXAYNT!B z6OM-M*n@Zx-5q}BeaxLE`)xa*4@_h7A*0E;Lmp=J-%jzjS7$~GVG}vv=PlI?o{B7K>22vvnCi;goPSmm* zS|%b28%?=Z$~h>vgK{yHYonZ#azV<)QLck>@s#VNT;fD+NCS^!Z?1Tzy(C%dz-~r| zCvZ5Sl%U*^WM@Xi zM}0^=bW}uq>}wk_U8m?nN(|eCN5+Ya+BIPa=HzTIi>Bi4h*njb5GNVpdpO?^D#OtU z>UR4LlfCC%;HlcyWHu5)eFN=xJj0ql68Fxy`2Ud}KbpJ$r#M%2BaMi2_y0T2bx$KX_qqG`#JN^Asx13uoa>Rs z0%YM|9GDwayvaBl=I+OptvrusB{fE%uxM9f)8W#Vy*!J0+}_|$`q*B5I_gdF88{hR zBW#6sxV{fX0L231ioN)x5c6-V|}-CJxw@_R#u=)EEp#b zN?tj6AU1y;*j~gIaC#g9r#RCy8KpF9pXs4dni(B?-VrrNRMK~Nko~T8sH+@G3+-AL;G=7DFY|l5;JhRu!ZUp7}^Y@Q*#l!jM&z-6UTdgGYj|QNZN0vA#FPH z>&;qcs}`g zHj0hcY}544H9TqHNgE~o?}W2}-`-I1FE=)pFr>dg_J`AI18sM~05g&{SL|Z97xl8tJo$A*us7zUW-=$0I+Ho6 z)S1jl*=D+Z8?jORS;fE9n(f}-;asX)`yN}rWtVolA48HY8!@YBchZiEeHqxepz~IF z80VP$@>4Whw!3R%1`7H;LlE{-t=@^&;VPVY{t9G@Z#V-3xdsSa2cF~r` zM%#TWVSt#Pr_e39wN%_n^M9dc2n}k6zgQM`+vXmnc9!9$?EnB=vE z+Y%Wd~!a~$@(mF#(&Vb3Yr$a}H3pON%V#ew!G)L~b};%wv| z7#4g&YHSRR;6KXtK&aEBeXZ@KXw~I!(n*Z=jEFCBLXSgd0dhxtYS5>c=pVLrzus=5 zKQYm78nj#cls}>l5Cy}W7*L`;z2?CkCSlL4CI{B1`e(xr*L*@}eI4G5;b)rM5zXQL zfO8|fka844ZlUYVigv{1;L75T32%Lfzb?zeTmNigE6WCColOcQX65^tX#|@8)k&rEdP# z6l-U`cHhbQ+M(Dlk^BYxYwrIW{3|5zs&JCD0Ac>M>ByxGIQ6XIoMYD(-&cL?F9qqi ziK^iwhTZodtP3tSlkURZ%f#IkNcoMjyqKHqd2kw%5C0>NMQ-ME-XEF5TbeH86j>5DEP4<$K_7Kj!}S zlNR{f&6{+8>+g+-fEt+W&^BX$484w{jNSAn&Vn-U=4alMZ^aKrM4*j)9adDdmyh)o zTU&{(_`LPx22XY?KH(r1aKZCYDO!|ui@#)IgWK1MDe#lb!4IfLKvB)Lt-x`oE(|)$ z<8AhuztbiEU966|@4!B7E3vbUYmxSv_1&)F@SaNhuu3S z3B2ig>Gn5lD!+sWQ_wE_U_X1TKWb;2u7%G@3W1>=NT7wM+o6S{&;kqWVcAt@FuueY zn>pjTSSi*YCdS{g2SnX#Fa|i|W1O)S>a9Nj#`aPaS7>m}=UkPXOC_!^iEARbQVgzS z&NY>DjVG>eh>NzI&M~;o;#`+OFu;kw|6|}e0^WyBJI@}Tt3dVigQ&H*Qvxs+RHY$Zsc6W zoNGC84F(sq_a1s%aD4;s?&0~Jp})V9NCN}@^5U`jd$pUiF^e+kE-Y`+-@d+5oDVh> z*4(WKJNg?G-e72r9c?puH1)agWoYi7-R{OERsnz1{7-e2 zUqW1@yDbJ+Gw0gDxmt)Tp14SNPa0g0bFSw(*GA$R1up3B^o`J6`Dt@>__vIP2-#6uRE-G#INn>TWF;|ml%cK zZZsHYamFIf_!u$XM6xEmjWM`JajtaEbsBL^AuiI}K!eN9xtyHKL0nnHMS5%hSZ_m! zhN!^MBS_GOQ;2IOxS+S}M(Azg>(HCwFGXK5_AAVErf+%Xh>9rxge;6Y=>Nu3eEcb^ z_*44PQx@^397d-O?tYEO`_#Z)q!y!mHlD9&ChQd#eg$s<<7OXiUC}88Hu%cxReJ4~ zm=S-bw?ee6*m!z%E|EfME()co)eT;}$TSS^yWzJdHV416VoUM+k=QESh|E*$ZAN7N z61#>YFT}3nnb%?)kU^vxi&2z_m~M4!8zCU@mX!n9ct#@l!D~&SHWwg|wmrBUgAX08 zSh0z!I17yw=NZ5EkEWLz0+SpbKMTeOcjE*bdj;$PV#Va;0O?X|hE~pC%ia{cg_llo zNL8eGNv%T!>u7c&#fNCW1urLOVqYQ#FMxQ*y0wF< z3zL1zu?y6Z;s4ddz-t#$C2-6*_|tlJBK9Ls#247c!tM&46N%Q}*rE0quRpV>q{m-e zX|xR|d?`}emo=@q*+nnQ;EkI--m{baW0GmRNjr>lH#oF%i1>DXNg~*kq0O&!Yk#O> zHRv?f>|+0op9eiweowKR@d}+TzZb~7@&FPhRFwE&OUI9a_ zBL11MrMZ0C|hQ~f3FFE#>xDR^!jHemLIJgkRC ztu$`Cli&6`e%pf>A?w@tZ9l$>+rEC8oL?2{qmZd6p_gEyyg&AAU--I!Z;1mF$qcZ= zvOvXvx`rsb!fv_+kSh^D7xaGjswzAmVNE?L>1ep zHRlm(Uz_f9C#9?Q8hS+yX3KSk#SL`kErWK*PK4Rq+TC8_cA4xYktSuV@7;)f20ZQL zjHMSu0mW7%AQGCMuqXV$ihIL$@G4G`!48;pi5tA{8kXQjhzxzx#Kl?KULP_Kye#*7w4h3^sl~XK+2kxnAL1LE^H33zm1@D^PB~m*nx2 z>vuHlit9J?W~Iv;8xzD0P~9d6wwk*6>##qeog|Oeg!k^bD}j5VH7tA#eo>fjJR*G4 z9pqlza4q(=8?bldMB11EWyaC#JonN|n)(1Slg8TG>^05MF?@o%6JJUp>0^7s(cLpW z3OGaiHbZ-RT?AZMf2tC{Ud#UB~xor{gW{QPjuL8=))6wx0#G%^%G<4H57xVE~1W|8e z5voc0J9A#tZ}su&PQ#xaM<4&6nGdNrV%=>%Gy_I#=za;l=y1}yr_+Z2o%zr|KGv;+ z$~>NRXpHk@t)r*;(Au|jQ~B$&%v9DRfvLPf@fy4pO{Q|;KVT}Su9fFQmBwug`E7Uc z+iEa!)<3{+`xgB^_~GAv=6vY?et*yU$fWam=pBa6@5C2XPFm-5dg5gI`%auT%C~S z!+O29??cw!|7^YY0=rpS?`;d<{R+B=L@h1%Qqxb2PjCGFkx z0SQZL@B7c_?LEBr3!g>a-s4V1?u2d?j;Z(`7UHTBQpz3dTGO?>1` zJCv#&!R&!9)Z#oGrm3hXO&6^zTEO^@t*BF~sJ{PC=O3IZThC8Ib@c;gIKPk{gYyd% z6+;Yyoh_lsv$m5@oulvw)MEhZp*Fi%h(*;_R}tyk|D-;}{L@S%@J|D` z!9P_FWrNK6zra5|^R(oj^!hj>DsrhlzT4v>DrT}(Yeglz0KzT^&FQ*iFJ;k6fHEa% zEcRL4$%r7axYH3HK{}Q0VsWSaEHHGGTJ68hnc=?)rii!`iy%?7ip8BMdW2}z4sh|f z)A#JI7SJASy=M6HdEBXu$DRH_IFCEwOJ4|U5#UCA(4DlU;tV%V&kk`X{ZL`gNZM4< zk3Odrm7yJIzk!0Hn0`Yq;hAvrHa1egfaskH@LI-x9%Y;-J2Hy7&H<`k^&GyK+PG0y zZ{y=kz1xvMy;ZQr`X{;K#-PUqFZdG_SKG6`CPQBnf2O|NEKWG>XX=Z`31|IGeepP9 z?#b!PL;CV0eNZvPlk_?1tL;SkIvrobVgpJ@Jsag_<@gyZXV zr~qrY(PDkaWB=RKWT^&`Mh$*-5;aKHinAzcPbxaL7C1RipKW!W%>G0nt4OoT-Y%js z!HpBt)>^-kv76){);bI4oml<{-ZHh`M%K&Z&%dv6{QU~Kvgw(}wx8R>O&d?f-V0e( zMEfbYeuDNh_Fvq7IyNe6-}VQi{n&muk^S-Z^UD6>_Oq&p_ESjt1np<+zqtK4QQE!z zj~VU9{>90)pI7!5x1Uu-w4cJGCul!o|HbX!Vs1ZXyTbmlop5sP=av1%?PpaH?Wa)N z3EI!te{uV9%&oirA2HgGeT$Q8Kdv$PSAeF{)^k+Y;OPWjrO;lZ2Ng- ze{uU+RYdzK^m2msGxlHH{>|q0KWwz0wi!;aU(Nj=uk0^wKdXvpKMRT+fBa+YzqtK4 zF52DxA2QmHO_7spKd>*9KZdH{TH_%$J@HMf3?wmY&D!*`*~%L+kR>O z8D3Aa<-6G0Ee&ry(C$Zg(KVITkT%@$g57b$k#UUkWGC?mN8THrA7TAYX9F(9)YzCI z(n9=w93jJ_u|s5Dh_hh9IWOt6{&(|fA)On- z8_(CGM*JlZu)An!fI|dWknHxJC3dJhIC{=b(6AlvKj1!k+Z-o?aD=A&2^Rj1UBp>- zy`YP(?S3Y@eMlg>H~tLKi7mP5zlZ3Oe@EZj#ruXvT@4NCbNI{gz;05z`axZ4x!83$ zk<@tG@K*G|bT*f#)*VlRsfzS{MI%0f*AD5>5kD@#ESL=Ffkvtt1-}s5W-)_fP91$s ze77(@7;j0ByRY)9iq;e1{>0wgq)*I^9ghviUt!@8?qL3jPVCD&5wKDFcIbz95U|0U zzZ66n66!>qC}4xHzENl-L{;>cChn-WO4Bz%T3SgHaNSM+28^su)zrrzy|+??%RF|j@69j z%n1hb6`a|{nG=ZlQ{LDAh;AONUDeffjDE2BJ06?ApQ1(jhHyHzfX8DCcngWo*aD_g z!#Erh?;blHqJ57j{S=ITa_K&ROu9IBLGJ|nry-^9J>akaa(HhyEM1RTx{+__(nXbz zE8QLIiSuNo3qjq&1r_~wU8=qBWl}vI38Z=+I(Yp-k}7SsGwf%(`uX25}7n z7kt~j4?>Mg?!yRZoag;z=aCTGr!z=!dhEQI_5r0hhBTF4_@3^dU-Sa_^f)?LcF!fu zCw^~#DK&Nm8|Il8syKzYEPHhq#QXOM^cq`+){HYZ6#4a!fN!d9V6P5)r|QE@GY-wc zF@+Czen^(f6#63q_I#MZX+&iBX+&gP>@*@W{4^r_doJy#CF~)ezetovXLg#PXDgBN!-w*D9_&KU8deD%!GQ|&&Gf(Os!#PZ)nA7Ms^5X&Nc}in^-hE9 zT+Vd`=lY4b#0b6$oj17rUN-rW=Rf-Snpk?E@7VKanKWkkC*zG#obd8bR&Q1^gPlJa zytT(%kH`$}kgwCxQ|aEKGGPRQP0dRBk(u<{404Zt=Xe}0~Oy* zTMQ@RLezktZcnaSPl@8i-?SJt20w$#C>VsU#k zza3vEL_H!+w^uaM#Lv>~$-y!H0 ze}(tNiCj-$=!BZyikppqi03C&-=0? znH_F?^?xw?_!n*W5oxa(fl{yq7ej6jhd*09_)}3e=Q3~-`D6SEfD|0f>`xE#C&&8- z;^CM~Ucfl6#bXyIf@=f5==1=49{ZvbKLUzxM>XSlhtNiLOd6yAyCbtz-Dx z5D;VXgMa>6?*{Kd;8;iQMuKi& z!^7%#lEbC5j%y9BzjChioa+GkO#Lq1Eo%nXU7Tw*=h{tNU+^yV6M9oHsL^q!YE&KW z!FYdyoj=&doSZng?ew8a%uMKeZHA4o?**Yd4PArlp4|S=GoA5IKrkL z4n;sy>|8`J_63u=hN9Jmx~5~#;Kb@``$rG`oCYalEkWl5F8GtKGG{eYSppKM?Ap5^ z=A*hYe*B}(b%dlC74QEDLxYacN9_t-AGC9B=!5rJ zJi>7Y>0=1j2m7j+ag3V!>_M z(oG-QR&=9Qc5>vpHR8i2619siK@6tOZ#k`JS{)rVcft6wEN5?2e zdrBAiT@VE1v6|!&h^j+3s6QQZCpK@*HMr(*t}@QmAJwXl0T+7rhj&9U8}5W+B>#zX zC-h+xNgqm;u8-4YeN1CM6wj~1bx-cuTp!(i+6L%j7o;bA+6LWykv?XTK3X&IMYoza z!uo(un+1Ka9&@Zdwi!Bk-p~pCO)BG~PNL6n#n8S@aQ`}!V!nWd8a*Z!y9_5*%->iU z$0|lTxq}|l@r16Jy(F=~&@K`T)`fUreS)r-tp-;M=lY0q)e_fmu9&Ogxq`!L*x-Mx z|AKzD%lgT=UDppkFfYcqS-O7OaNU!871z&^@ECWxzq`jsM6z?y$k$!e$ zBz>-TBGzAyc#>{|e(H(sSp97G;JpI-+QYbX1b<}IUh^{ec-NuNe;H~*4i>=u8`l-h z&x7e}O}ZKbr()=;5<3DX)>R}c<3zejeO%Yo_~lGj(~v+{<>)2#8M>~{HMq{^T$gdK zC{(L{GS}7pcR^RneB|Av`J2hVy#;-dBcmt=oqJQoi(mm8KC;t*&k{D!D?|_v_wrZ& z%_N7(JI+?K)3i0-U59)gI=N0o9R!-5XQn`St4LV z2f^qYi|j;L9LsPKOg4+a!Tsr2RJ@x%okYEbiJ%la10;f*FyZ;qjki%Uw30-K!-fBN z9gd5H{v2SmVj-2!{$e4OL}LhKuPmMW2VE>@FJof43<<>Y1I2KJSnfwR4K7<@(tkJk z#9`H)fps;v0YCo4q{;?)xl0%)zHJx02o>)pl~H7uOe!~E zcYvglN2}}{blo;uX&1P)DNH`tZ0jx`OOt7Q&O$hHtP$=m;xSz~UAHmeIF>Tue1<5H zE}V*L2q&is{Z^WPIk_O{@30n7Ounb@@)^bvrwd^;34wVDCo0%Y2=QdIObFLudm$_Y z5&&LP%8Jc|g=D$4>k+k>;({a$D-kjN9y-pe&ss>KyU+je5nTvJmM|fl0$IR9K0-)P z7eXnVLvYq|vJhi^*Udgu-w~VL_ab^lvi6SeF!I}xPsJY{^>u>SOBWbCAd*-0NvE%?#OVufB^R@8DCf$fPE{oeFo`-(LM1D7SV1^DNdoT@=fQ zKVt9lW#F&dlG#y2C^5MH8Io|58-L_jEY07BZXsg+=aRmoSU#)_uN$V}H8FZERZGKr zVySp7RZDYG2GO82d}3wPH{?gO@A!Q$ME9ZYJ0WR z7g#n9-yiF*eXgax#>jZE0eR-N79E^xhZcfWQ zhZ+Wd;8MKjd$bU(fRg zN{oCO78PeOUd$QCaK`tDv5Odg%S-HfNN@E|i&(2q<&5>js36?J-gEoN;QEkr1(Bf1 zPY_oGanaiE9|qUoIoAfxz~J(8u17eRm$-U?3$xk?xSQa&w=sv@Lw`eL zjWz@MW)>Ywmc9DzYkyhm_Uc#z|X-$6Rv0IMf9cwH95`(BP zy}bM(oEBbL;PPYb>c?)>wrE zc2)z&T%XBp7->|b&Dus0X+H{rN^*qe(8ee!p}*_(@;Z70cs^L&78vhQjL z=q=<_?M>CUm?XWMDBo=)!jFjCKLPufDcHZH<*3`YoL-o z13)iX<8y=flp#Df?64tt(NoMZLyAry{H0gK$hj{Qt_^`tVmHH8GnSaqJKjN zdr1R(<0;X;&R$F;_6+R)?Z(7Fc<|n6GiC-5&-p0JI$-tL6U-R9zY+z<-hFI$8ZA4Q zx9kceX#bP&a`kuemaT%H2`*p6XMe}^zbt+xx2LlQvrq*(dyoNt!;T}*Akm}d7qFUJ z9X*J}ZO0xxpjj*aj4Ye8-n>fJM?o3W$6_R?^#qE@h&k(LOHk`A3nhJo=OdZo{4|bB zNZ%OI!@EyCPLh*O6Tbj$8!NOA5Ya!8dlUzXXJGWg+uP})z3gKp>})~zqZT9C^N#QK z!si8!Bk%S#7V^IExbl7hw;f;JNAB0<-C4@*|5hgN4=EfYyGWpov4+=*`rRrm7$4UI8Tw;Y$HYDLt~Dpk#p#;)j0+@k3Rc3CK-Y25(f8fZ^ z%6@Q2$@;q^$=X|K^x+9?nzK_`_$=K(tfBO!lfyWng&{ao@p^=Nqz#oja2yWfZg?B+ z{t~zawmN+o!Fw`670V{1iAWQXx{$h%sz}v#{B;WUnWig6Ikb7c3YAXP7G$MrdH4** zve{S&*|fAAcCL%6mNt)N`%|_63!aO?PYWLWg_YAWIiPh;AjA0n)9<-U!_%-1e6)&q z*|J8SR;m7|^M9i|w)2XZV~ab#JVx8a^evH8t#SswC8Ev9K_~kaHDA0c>Cx6zfs{wn zvyJAR$(wfx5(t6py}p`+K#P}y_vk`6a5ED^6z6(?xN3A!wHjRSaIVjhP+5rK=%Px+ z+$(rRNze0%X_;iGPf(4u@0nfVtJOHx$-m~Ly`8Lmzz%lEOPcQ=qMj=43in!|n`0cd z8^|0H5D`Z@CNOiPP!X9U1xnJiMeJiQ%)V@NgmA)a3bDnZ67J>+l}Iq5l;VnTIr>bb z&96)4vqC17?~tIqx1vMTKdwvV6@#meb8X>VD{wHh{*T~-L%F;RQaPho8b8H(N!~xP zE5pTb4;PKmJM~%1-qj=oX=FG`%4B`=!oTZ6u-=hI<8870_+(fP=`(2dGK|T=>SJaw zjZf~@h44`U6T&x0AcQAyg0p@DZ{Cz!(Yz}c$o)gNZ~hwa5?WJy!x~5HHCk5uiwPdz zn8RXnYzj`^vAr2ChOdZ|eFUuL6J%4hx8cm-(%_RFDS`gisJUCzUXvW|+->+v0&EBJ z5dGb7cP`?$_ya0@i?a`%hCJLJMO!xqRQ&CA%@&ISoIJ3LwE}&>iiLC83%;0yOY4+n zzhnK)Uj1kGe522y!_gn8GIcn-qD%`DeFCoA&l*h^-DC5a=++{E=zOr?`ZltkQV4j7 z!Bxq*G|rX03S3_j7xmCAgKH}1%H>>cDr+ZkQF!5UgX>bxHI8$gLg585mwN)8GZ?^8 zz;5yW@OUcDtBSLaQy?V1K}ZlVT&f^Jd~g32M)1hOu}H#_&My>RX(Hw!^x zNjV*AE#!)~(X!Cvay5Jjy$N1Yctr1-dtM3MPWAGYxM1y-3`^ zNnu-k5DM7NQdd)N8`h%sLTR}z$96FKo7Df7bQtbmC=b>mcdLtW@|K3N>yH`6F6}|a zVm+&Or)%%jMLT646KxI>h&F~cGm>?oU1D$z<6KvAu4r1$3!(j?2tvEN;8=e}=f`M1 zCEW2oR0?Af1}p4&AZP*! z0bh92`raDxZPx__MF@!G_x+wbGrOAvYWqC?pU*#^Pj+VRJ#$~qIrrRi&pG!_JI(2{ zppEHIt` zC%AwjS<`{xKY}74{zZ@{(Vh@xYW8$EKiE&c2=4+o&%Y;q?Y+V&HS{ zT8MTgOtRkaW^ec{@qXd(Mj0j>k8sGx*?1%QkdNM^Dg+o#-k1o=?6%H{)q=aE@6Jgd z*6DWmygW{5M9tmto-cn4R5O-nG?cN?||Ihg0x<+2unCp#5GFI7+9-9uwmy(>P-E0#2StK9xFZ>^;%fx z2B$%6yR+qR zV{~g`P`>T}Zc~6eZ~+UPPQdL1T&w{9>;NuTfU6W>Hv+y+K+Ba@zJ)SAf?jz^w%QI{__M+W8LPc?$3X1^6rh-y~p^XfrdqIe;z&*h>LENWixU zXt~n!M-#^t>=(Laab7<~l8P1#F0xA()`&h^J- zq!vDOb^#RGn|QC*8*;|--;L7k%9(*NrjEophVkw)wRe;KSl<|k`GW$GnE8*9Y=i>F zyHg`7M80_oW;1)d`B8)#Yw>Qf)9J`XY_p5N7c)095Z-D;PwSkVgOG3WW8_pCRq(ZL$%im#$NysT5z|Ix=e`B;=Q0TQp}BT{e>X|`y8;*V zcP^K$7F}}K#zQ4CB|7#s#@k%VjT@bW=(iwZEyX0Xn=}_|NH4e>K)$1zgnTp!!34z4 zbutpDz_GZp=M25W1^|6>SxPeg3wqvk-9*qf4oh6I4=Q{$k3pGxVy}JDgvY$n}NbeB4VxOvhJbLGN0fsz`zKdy^kt;k@oq;PnQ=M@uV*kRWd zq+`Za4t$A%zd^wd=Q`P9pY0gY(Y1h2(WlghZDU2Sw#t4YCw}WF{2%JT|35>Yj>7+V z;a~Cp9J|B6yG-#vF64h$PVEllumguC{~P>|ptMx}pXl&EoDJWX|Jk`C@_!P)OIReC zoKYvqE}&Zy%tU^+j#D3n)=`X1#?r|WBR@w?WTy7ZChLuCyrJ;OCDXrTq{U#2qFZge zjxH9w7UKfEo&npdjn`EWDbYtKDS0pS$9{SYKjqI#*w-z$P&AA%H}G^~;5|T|6S49l zTn=i~F=*__-!kwa`V2!4$BDDcUfhU*xs!wC?eT#rQ_i2l*E^3SZzS^a6>E_k{tn;W z_D{27Wx2ZbSzoyce;IAAW)C1b?%Rs$5xw>nxXu0^!$G(Z5XS-0NXda?(M59LSagvb z7g~~Inn;decs+iDwQla~E;}woh%DfOSwaq3zqR%qi@HhnYZGK=DOfDG7a`UQC-a4H z69~UcH~V6}eAGCP6k+n79Wrx)-)4gmmk1m9Z~+_qh%04FK0F1{5Pj(K_VU5@$0lQ! zJTVdRgbq@pWmWAk8X~(1PyCsViGKs1-sU2U8P-+;F&%8n9r;o}H8@L{POQ!Mf z0tedsnW0@;?I9Fegl~2D+q+N@z8L*WCHs-IGj{EzN&e0c;9gKt1MH0(@CG6kaRcV? zxC?Lt?m$eWVsCDE^mwc$h-YeVj109eG=g+p;;k*gL4j14;c7WYulJ__nnF3>EdnFp~Eol#8L!Is>$yAS@yw+=wQvI>|iDP zXZxEw_baqe3`*N+frweyzP4UgXsD5tx;sFG{3kQ zatU7vqO<+#-*6o9(;r9EXdssm!m7gtn+-s)UeSdALbKxwL^2tWyP1qggP~6g(u-F3 zJYo}M9D0odHBCX?s-QfCvN-g;%Rs;l zuurM|wMp}}FT7dNOD4Mg31Za}FA2eIO&rm?DlF>Qv&JLj8llYw*z3QSj00_a;WCFT z?PMJy_7a#$mgK<5?Kj#Oxy=xa+<^-)at&93#0cp2B?e;j-3jQcvp)FuzK;R}8sxl} zHnjcT@OKDCxA^xW#lMrlHC+4wJMtM_D2eT60t(R?h?N8F6Vq$?xFjL^+j|tQ^e~nm zm|t;)W)4qu7-H1CP{&mlPQy)c_W-OYN^-CP;hM=!1!c#>S(+4T z$uNq-=aMI1G>eSAd9KCS%K(8UQtz@DdqPqbMZtakzDiPs-%;lQ$?JFGbqoiQvIZ=L z*y7`cnSdJxU#LWhM%Xf-KCLWkG?%6__`Voy{ABPIGi`GI`640b3%G!s_rqYiYK_g{ z3mvGSg1T2hl@iKg@Pn6t=%0+U^!FtGR>VZCs#1v(#QZIq+6jOE4Z&+ET2TWTP5Ap$ z80^0L;7Zl@f0s(Cvgi%SJBp*6ebk zkAcr4P}tGpxpaa2{*g}c8jk<#Y@FPttMQKuaIy$OXw_b8{A1`x-!+oQzt{&7n5bkw zZ8wq}Su8f zyFO=*94Qn7@1iYB5f$q;6dYvP&NZ}=C9YoW1$;mbmHghWiVKK_f<;1gdox=O8 z!v9MZK9z-^t_pvch2Md~lle=ILx9uAJU_bSr1UgzX}A37{BGX-z8lIi&_OX@ZJuHG z@6|EVzklF@{(bUG6dt-jT0V+}pU1+J{TqyTp`9fB?ds*#UD5{ciylOr8|GzxJ@8kk z&+HnipWmYmXA=0=V=8yPAb1tcNS8bH7OQ%vsd{fns`q7xis-W_wB32i`0yi1MRB3r zT5Zw-Bm8OfF|@MKbh+W|tBvOZro`>Sg5`?A@5N7M>y!L5dfR#(-tYWZ<_+r2~hjY0ede47eQTRL2IIw8e!(T+>uc7U5hj0ze{Yn3XDLVR_ zJ6So;_GsJe4ID-Qx++Io*mgXc5*c$Cdr zLijq;^Mqf3!1f2K={~|6`}~ig$}Mt=P3%k@e$Sm!NU!(bDF#xHd3~{A&MQxZ>^TSC z4Dees)L~!&GZdWDFf&a;m5dob6BlfI(}FK%*{ZZPK~cxD%mfVU_h8SREA}X$Y{Y26 zY|K2V$iZ%{=zFjP#@>VZ$7y?Sr|o+9^XT=pHmXbrgU~)LxDmiOs;k@qi!IG$JyzFK zUmsw&#=gd}o=0e1Eu3bG4@|^J(w80o4vxvvON))t zUrsVgZ!a-QOQ*m;QLdX4vUO9(X66DkLr2?6sC#@+SPTh!Y2hW&uFb#-oM^O+urRbc zTag`iILFLdw#gKOpzNwwk$9wcyKm;*M5~HWUdW3%K+jMKGlg$|=ez zTvT*?;fA6dxDiX-1>owgY>19;dK4jv9;TvfXhyXCfPFpIrfKX1%0t13T7kkXQ7b<} zI4#J_GLJ`@Il5VdcSmQV#3FobH>WEjYcCkmEiLv(ypd?<8SUE%ByxA=rf3f=B>(r@ zdCN`Tqn&eqifJ(17F`HMn)rjv=!3(5gbrxgmn#(!tgP(Vgo8xAlI&V=Jf2bom=WHA z*M^a8`Ut3!L0Zs_Cm|;#2E~{Qd>HGIQe0Yx;wTSqhh;eS%7Ns1-@;Ss0Ja%`TE*NO zrQ?u9m>H9!EM&=0BDoi0?fY0I=@WjE^M>%SFwj3^r)MPb`#DKNDq8v1PPqL&IFl@q zl{}z+iZ&`arQ}ld>kXtb)COVZp(skR_XpsESPNnVjx@%ud$N;Sr*&TI+E!JoEtnNO zbgULIb?Ih-5`*SX+cS8Z!`>rqV0Rh^*NZjySm4HrXU8!I?JZrhKk`_4A?a8yj7*pI z9NfQ<_t>`#NwFi=l~EI=zbw{EM^BPLiW}&(7@LsoQ>;K5m_a1EsV6>cMD1gES1ojp zmX?-ZglXw%+;}nL4ezFfw&GrW8gv}w{}{;saghJQSf~x*mxY($8;g1TH>@M-3NMX* zRSzs+LZEneq(>b)h<0F3p&^h5`B{_2gcR9sBa{qw(oBU;`JMtgh5|-UK=2gi1Z+t- zF?F3PIQr~*zQGEtxE`>EA613lm7@X-GzW)N;b;!|cxt(v4}+%CGa*A!jU0sXQ4U@w zE-WJ4o+F$0v3QhWx4ln7k+IA@D2YwK3Jn2u^~k-Jp?w)GiL#`;2VMlBdD8rLq%RPk zDE(RQh29B$afQ1QNZ=%gTEO4Y=+pRMQ!%;VW#NLi*jot6b;! z9e^p#?5#(79+QtI4R>u5isHbN9@OsSQ?6sOakiL=Y+q^C-6+mb5z|`mSxiyXBx} zE`WyR!~B7Hcb66%gSX7llgzOtftId;54)O|P6_baRTHtjy+5XsW=~Mg;AqC)dooNj z_T8hXG$1JtWXA`vJnnuT++hY+;sWC)dhttWUM}XHL_hwqm(2sG1Ftf)KG(zVM3*-L zkjD}v`1%j)j#iETC683zZbohhH<$`ZglAE3Y2 zrA61gf{tNmSi%#^PV9xR(HSQe4=xABN;6vMlZn$H80&=yl<&0QdZ0XTpsN-< zC^ucSP(Q$WK~-p)7|o3Ex2c7IW>gy;xf-PU4&Z2y9%${M)f_;B!1cwLiJfIj0!2H& zYCqN4>Jx|aoiJX}q3V3uCH6!LyZjYz+YP|LME~?~EPC6Q)`S8k67dqd6>x*=WsM@> zNwNxfrs{5u;CG`t#jynGgT-5H9LtZKGHED|`!oB#?lb$sg16QBu9h0V!Ao6SJ#OXj zMD)fwIVG8Cu@Z>+#PMz0G3!DE@aUs4edM#t^-SS0th#)6`ELSHG|0S6qyx6mOM;Jw zEfpb{*`*{x@==1i93}V&;8@RYJYqso#T4KzAA$ilYN0BopbXSgq-&wscvQ^wd2H&Q zWZ5tUPf8NYRP%Xs3MS!L5mFQM8bKI)qLj5oGQB=P9IHbw3IOc%nUG}pvpkG-0_Nx_ z)cBM1V{p+NMDD0ILOK2-wt6?4X=`Px@RZtw)L=EZ-KV0d{y=~eq3Z@2-kT@w9Oof^E5 z-Ha5IY>~Z*1!B}(Kpb&-PB~+#1bfg{P7pqhEi`H?T7+Ou`OTBxeubvb8J+&~F)aPp z2M%CJGR_Zk6GkNH&9fP;!&r z!b=J#a?s=!UOFNAujrO~UNPeFiWQ>(Fv8LbtE6IQD-VVb{F zLX7M3;EpdyWJnbp^Tp!{_{*~|Ni4z4Wkkni+590&F6-w_f0?K<#}Y4s62qM8&h$?g zywKSM&VtT>!w)E^!k<|vdEi-~~vmg|#v7MkqO2ca@C|Rs}m=}lUtO{oVNhpze z6gDOR_mt&ihc1H=xp<2p0ua%VxT{8&$Kt#A#a&y~3H92KA@>&2JQOL|V^F9>O6WMq zc!WCI-}LK+dl5N0_LCqE|HN2t0I2BVuO&$fxsW13it00fy>M?ysd~P0QY;Gq_UD2N z@ufc=P~QsiATf*1OR64FSiO`cFCkGH$3(;@^Cj)c*MWvta3x>80+K^J{(J(4MU>8F z;30NkCpx=N@q8!?mX8>ZCN6?JDGKf5Xepn66HIOK6B71M%V2ivL}euNG-g(o`_ueU z-JDcjXr3>6!at+Wxo8;e4#Vgc7)G1@n_!z&ZF0hGigNagXqOns1 zL9KdKBiBfdpIDRGcI)iFIIq8E5LCu@2_*^0^FzoD+%}@lI;QKV7uOS}G72wVZ zcCSAlD82p`7xemR0@f37yaHVB05&MV?F#S?0;Ul?XDh(pI)L{pz{eHfuL+pWUQ>?) zt#5V!Z&HAj3UC+!GYI&G0@NMAF$!>^0@MiDgMd#cz|$PSQxxDp1^6i7Apl2xodl7Ju0({f~{DT5~S^=Ky0bpMO zc2|J6JAk(-z&jLR&S(IhN<97(x+vOsi32!60sdM6b|v6x1hjT4^mhP%tN@28!0$!@ zFpq%NPK7iF@G#d0kzSetY(^Zhd`Z{dsqoe~8$EwlfLu?Y*T=K!SuD}osqj|^aD@U~ zs{jwO>YE8@?NkUkfB^-#Q~`cKz}WtgsKEe_$2fqQ3h*QaxQ&1xf-(}E{;|*Q@{V%_H(%p|E zG?{wW-nyyjk?O5rSZLdGZqm*1*z`0iPY=6tV1>%bT*(~VWbs-CSFe>_ zp{H-tBNO`=5d<$|64X4zW;|!u*m~w1!B!J4z}AC+TD2A$GC#aXGd|OZ%!Mgx#L;?O7#X4!?AxGeA<@iKul2|* z^txWxJYVVIgt!ISXI z3g^-;mga;po_i3pya|U>8MPO?W{bGJS}AY52%BE;1s-G$V3e zw!2$e;hI<5Pz=W+_%io}DV|Ir>4;R0XtG6}l`3JP%2$Two%Hd3N!i2V$0|!TH zHH#qoe5J;{X}k(I_5$U=17v0l}Ors&slYfjBEG1>Qqb|~G?^)f_#%fQx zUWN`iiV8c15PteW`rUSRx3qDkS$8B`#BN1H#Ud6vUCLO(+RWpnU4bk~s$|Wu33DmY zY<-|Zz^sD~;v(l2RG^#(A+#|iP|jUQCj&Qzd3CnOJO**ST99@J1hgP$8h{E{;V~`t zAbzcm8s;+9(H!W*!Li5)90agNUl1W-p>MJ)Z@;KeGPpZM5#JQ-XG;P%1V-GL^fHm=3FLgb2`zj;7X7scy(KE7r%=BwC= zd}9@`ofa4dyg!Q@4?b{H7+7~&VPHo~k@n;pzS8YjqFj>a3GZn5{4`Jcj=+w?I<}T| zh0m45Fw(a-?)cmOb^AxHjRkeEC7kFf#20rmjcO#!}0z`qgD znmac;fUhdRf8c@=?c17#^9rgPCfZdGp*Wd4s~xCH1$CE#I)KS> z)$a+#xwFK9xR~`ZxNe1TjlR&25hqT3;AMLF6PaVIE&Y?k#s4seTj`i= zALB9B;X%g^Fh90~&(DNu;6o&s-68W!5RBiuef$SLjg@?18uHyVkRupTbjs)y%o#DAq8E?cB5+&M){ zJmI2c_@HQh_074@CD+uzN=n)yb4<02Wd?0!N69DiLIs?3nFc)v283w&8 zx0bz%^I?R!eBm4M!v1J00AO|tjNFW4#k*+LkDx$vH(_4(|6B%P{P;wB^|l5a{^37W4{z2R zzUXS?e{svs#vOb1WB%TvH+&Ii~w6z81 z)pw~F36BmoN49j;BYg`B*YrWf?eH_!9C0=#!;G_431d)#H_!+R>PKZCNQl}j+e zdT{i3WZK(FOZdQ!LxVSQt8-fXq?CA}K!Z!iry7E%fR3uo=){+^Pvp-){&?|o%iHnk z*7tG38w)F2Ml@c%YX1grc$2pw)@^(b*QtThmj8Srze^|iT3{BM5Z+Qy z`Pqnq$}L?RcQouj?8Wzv8lv6C_88F^yO=FBdww+vbTs>io1?SbNQewQbr_hK!2s*P zzBBv7in3gUe8xsV>gF(jY{(OKe+w6|`_uiv?n?@V-A4dE`s1@$a>o4*{BIQeV+uY% z_&+N6EW+<+#n}Jrz)x53a}<0I;jdNj?}PBsW(7aYf&ZC;zfi${_G7?br{Gr*{yqht z>A-hW@FyzxR|$WEg0CR_HGsGDs}y}|7{b_~wPNzg)Xf>02%Vw2G{_wK-0!=m1rB$S z4+Ut|yzJDLK4{$&3KJK|~&>KEH)`+;Ppi5ECnIq@oeIv2}W9dtb zrO#TULxDja{G`Ux!D!Q&ZEd5?;8|&T7^@gK!^TDypOprF4zxlIsK)>uynT7P|8&!J zmy{m^(;c|wc+n#ocO2Ni3Ezvv*vZ-u{g$}k$T@6;zlzR7cPRrtM+qG2H(<=tr*2R-K5 zNQLhXkv~i@5jO(t<>3bF5jgM)1BY=ozQmG1*(==)`{>J=Jn4G^IPRH;PDra9@e}%o zJ&|01BhG@6Ez#j$tEvjyg6@?l7sp-Q6m-|&uIa$H1vpH5?gI6wx`jZy;e)jZKbYey z-4`Fi>OAI@ent6v{a<^ApRO(D=|-_}_>p)*mqgh7q|%TsnPot@w9bN z83>s-##`D5dwN5r=?Uf9?FD-ybW!M_)1!o_0T2?$_2H%;{6vvApe3Qb(Ffsrh&AH@ z`)Yf#g57Ne{p#3dtf_`23Doe0z^8`?Z-0qdn1t>9H+o=Q7ieBu@KluH0ekzx4G@xD zePHjNlzLkz&&GqZjfjhfIzfDIbd51x=}CaAQvl;nKcEt3wQO7}fK) z>Dq;6~(v9By&GW`xWG?+Wvl0bjc7(lo^+pzMVo73Nq(e>I zi2qU47p>(AS07Kh-tv|XS<3Qze8sb{%I*~5vMzpS@)9*^ydwSm06LRBN1 zhfrfg*knu{Vb3vF1Ful2Zovg}%tf$K+ZKhl&wz|xe=1&0|4NUH+$#3mp?v^>2`k<= zMITJ)g3^7Dd5xOl+We(8%cpSfsv`oZ1VisOfL2uxIV85Ani5 zG{R|7go6`74H$0SPAASzIY`(@FE&n(ocFOjo%fDh&e|-AL$8-huRqA8=Q7F5cu+1q zf5%IBKj#P9qTq_XtPo2Dlo&xy;U&1Kv~O;ao9MN~N}zE<;C2^k9(l!2I9H$dH7;81 zbL5%Gd1pgq0&+pWi_G9h%JV;0~zy4)?MJ^_Bm*V38}WfvgaN@t{_K1?@f>AG1b zV?IMTFR&9vF^{`IzGT55i9yiz;iHt(!_j2KZ>Y~bydyYioM2ZKD4&p~gf1qU+KgA{(;7hZ~+>2f2;o5)*Y@gn22Tiux}2X%Gaa&q)m-e z0>gphd@kN1XRZAi#`muHIjQS2ryHfGpc`#XdTCcZze$U@xX3d-@k|mz2O+YyzzhWk z``?8S*`S7wA+V$)0d&q`1;Ba|?lEzLgGZs2iUgOd_Dzr)Y_pwrZ)ZYF&cf#coswqIXu^Tfl{??lblc9B5ZZvU1k=U#)J2Q(bwx>X!Tv z)g6F$RQ0Y$yXr(;{NL`~9Zq$>#RQRz%w<1B?|$S|cW!ERmCL0%C}5+$gJAsMW6sC> z+W1`cBh|rJ3ltbo0{yik_Wwr79`648Imb(wc37A zEM(gH#9&?Vi`uQxk5yBAyy*nC$1CNQ(QA~ zbthWNZsxI=-=mn+WPbD{TGJsJyD0ZWXR%TVZ|4cLMT0ofvlAVan4~??i}-R<@1Z~J zi4Ik-_R+)gDi)rdy{dKg>OqhnwLwvnS23S-_Uape>QL)T>eYVMt0I~@Hu!6*7?weX!4mhAIE;d6U};Qt$wcEaY)|wJ3qnssxu{ba>6b zEW(}7T+12AXa0#ZgU<{tlgnS0;{xerfWP}`!H@6+^%Ls!se`vC>=pX(;Oo@K{3G-> zF3QTjFn@~h)qPs+S};0L-P|N2mP zB6`_$YR-ixZ1{@xQr}pf-kRTl`c4oH8Gf7Ery$G**HeNy_G*3j%=J<~fY+h^I{}iM zpCQ#_E8+RdUf6-3*_7^9Q$l56^$rji3YMt(;6(M6x$6|7dJV2$Vhmr?vAtZ(vAtb7 zw$~V5k<#7|AV@oHZ_gvN*J?_!H05LeO{ys;11+*1GloCfu|1%Pv}ZOdYllB$_|%m4 z@Z9SEq&iC@zRv*U zKWU#E?UPvfbH9~k7Wlk3ssjBiejA>|ZxCi^&-}4M-xU6e{~VQfR)6f>gEZLS8&b(u znb;{g4q7rc4X~HyH{^1i(5o1iqHrs>Zlt_4>;#MqGvfgVK3BoJae?EZ1AL}}zt(}j z<`^lz92cxSjz0lQ|GuLv_t9tvs!%~)qM&jJbqb;AK03vL`jLVfprEn{bt<5+(7fw- zm=P+DlMTxbeUsMfRXe##q`3lTl>8khNLbMl74Cj8=lm2eVp_Tz1wuGSmu6EMK$R-y z?o7)IBHwz7<|B(B0@$h8aXh@85y zT;u|wHOJ+So(g9Y_J2_9X~E-Ala8Il1^nJt0c^T2LQ*Lmioxwq4I?ZLHLTl+WSM>u zFKL87j7C9JJwK|2c1eKZXVIT)jKTg3;SdI#R&{_{pceWL*@c05F4!u2qpl*e3^SuI zYgEKFnC^D?Rl0e*V)q_QwEaBc*@*04?50o-f__i+?gEjyLc?d6dp%cWX^N(KwwN`Sw6EYS8BmSUuY3H(--HqGet+r0IzxF5I$h( zT{HswC_Q|b#5Q0VHVg~A8G5iY1EDbp3azbN6MRt+FU*Alu$SSGL;JRoxu1uBC z5+h=#7ZiJ|npPms2rnn@aJmsn_plj-(AXvy@S&d({xuY-#Zk@*x>?u{p5YcKHH9?o z71D2Bm}mCDXu2c=OfL}%kzSwjUWFuBpEW4!21REN&llJC@8MY;Q@Gz{2A1<)?ZU={ zr3m!ynb)5KHD3?A+~Cj!IVdaAYiD-1v~WzVxb6eJu~zoxf1!o`c$gFryrgX?!1m6- z8X;nu7Mu;aHh5Q&ISpJ<<|-1*UzX)H=K=FrA_MDvedd*dc^#481AOMZA)Jk|@bv-d zWgZ?5G|5{UB95?I18A;r&+@?dRH22sg0@iXk-O5$)q?!8N~A7rcU#*?PJ7(Bf&h)6 z2wdpk-)8EX(W;x>^>EO=3~%9O%2O7t1-twkFH@1g%QC*@zjoQ;YCi-Sic0v+RQDpjcdVYih$A;X%(mYj34U)4g8Gl2zQoRFTc<2;Yol1` z&Y$slwxmc%j_j_uFDRyfgG`{k2}2e)u>KkDDYz+s$k2`OvP!gpy;#O;co7M~N0Jeb zBpDy&8eYUyO9(MlfWwQHSwNM%h8HcDM{aHidty*(bjqDQ2#L|2)a8egN6*tE8Sj!5 z+LKvZaerev$ueXK4?XU&8$QE4%%$7hGVhn=Di3BnH(7s>eFAWKP#TSZIBZ0{aFjtP zZLo!suH;bvo+5{)W+mj1cC5%DJ%5FW94)j2$_+8x4^9CNPREWeG4r}=2-s)+u~Jqa zlqCv9wCNa-2eL22?ZEUzCgk2;fPzmRCBPW~=;{Dy3gC4Bs6r|M!-=|ICU}slG{E0D z(^h)+t2!T5Tif6g0w{Mo&H!7f>nCm@RHKEaf-ME4s+^e+`CTSdY>m#=6Mr zOV@lG&G{4b^sGab^Zhu4@pCP7Gbzk-O}*IHhkdgQd(b@FVv~_>-U3I)$Q;$%`%ww4 zQS(O`k)g*T4fX|_elWs+t7dk@b|K8oEWqsNYJ?{32_FXEmEc{mGtl(jT!GJi?$aZu zd&_ZfftIs;ts!OHZghQi{n zp<&0v7lI)|en||GSONGpNq-IO#OaN1h}(~v>T6Vl!*4}D&p@}SKYl{506#7@vI1E= zfki2gP==zUmdFC##2OQw;3b-8~t zLxu7;mhCFUL4#WGZCdO!FsiE1hYI!Z%V^{pP`5)PgTDkM_a){@OA(aY z1aQuUZ0!KNjMGT!qm_6Ud2+Ik7Q6)71h^Rpx4U(!!oy^M#eB$>_DqxZ%$aemKXxb5 z^tzeXUNggg^~^GA+=B=*Y+72(kE}v-Jj^p%?dgg$KFXwgKIQHqK<=)%$C~e7>QBtT z7kLjv4ESd)pi+2dI4tYrnG?EOJTo5gRUfKNmP8zyGvU((mamq@Rvg;(a(?70UjBWQA`7ISF>KsDU;MLeG~>unCNsSjovR zLvlI61zKI&oo9e9$=1V>XX91&ZYX$-r0DET;UC}={^)oX{LTiJa0j@-CVz5naCpV&l3!9w8uaIl}_3zo4zH>EIRhsu#=f0%Xqh1V8De zGcB?lN>5!cAE}WMz0JsvX|*NnAqZ}ZmJiswi58=E(z_TIgi4NRwOeVFjhw})6P!>~ zKt_p$I1m!!WpF(NYG|LX6}^eiNP9KUtB`GYdfOL{`S=vc(0)Wl8+sfsng{Z3Xm?2& zDHqLyF#z*E*CyF?yJjXv#$AX#@P^;U(3l3vnxTAj!n-|yLEXHEa3UaKm)YdRw?K+)$@Jz=D#wtwiHL! z`%l&pI+DM;Y*+lar2Z2ap_H4@~1ay;>k#ad20+t}e z1wFV1L`P4lX`-h)@SD=p+p$`;qX7C6QRY1aRh=D2@tvU8{}=r|6|Dx9PvJ%Nx3{E> zl*`fjyR47&cTSr0m-=>R{WV?aFL@fM89>yS_Ym~|tiJ=%YV`L!UQ~b2mXwilIa+^L zA^-)QsA)S)`f_2>S$}`m75$|R7N8@D8uK24{-5>t0<;?a9m$L8@6RMPly(fYf!m-P3s z@1(z66QaLEu*gaozb|0bw+a%C{VgGC%zFr0XyT9vt@gJt3@6JY59JZ$)iENE_LS{{ z1ma=DYCVJF2-=W;dASyxh8NXdpt4x(ay-}yKwpd<1kKzFv4Im8PvMH)vA;49z4Ss% z>Fc$~UNxnEz$x7evs5J-mF6EUZD(X^9s&%p!aIZotO(j==%&|PDsV=4fTiC27&ySO z9lmgjm-esdZw~anyxSKHB4PIt#U!s=HMv!4rknwktFpg^C{+D}(d#c0z5S&TuAZ z0mUS0wt_*iGy?#}#86(K*cLz=Lu@<98{bP_Z`2d(B8CAin#cV5}h z1k3-sED0gjhPxF8qape#7|7FDEuK)WCl|MD)OuAyfv&IGS^k$P7&?TQ#43~flZb~5 z?8J%c>SF?k3cWxoE5I)-Q}|V*2z8Ii#Lu{X__%*U zdaxWdLNg7mgQoynHQf@l@gXRdIQx&5guV=>3kNDu0-^BDw-k`*q0=N^jkr$o-&gr+ z0CkeTM&+yV&`JJ-Du3CL>aS4wic>pzzeMG)#QTgB!3q_TF<3#i@5d0p!3~5HwCey0 ziCQU_4ne-=PheA+xC=a)8tsc!f{Q`M?;{H#zt1OyJp>zpYCG;FwiXXZ!S{+&1)p0{ zd#Ctbtn%MJQvOVp-*TjUL*+-0ls`b_??(Pn@#O-X!dELmeWlYz7-4ruwElzfLg4@{ zTb<^wRr&n<61z)V1oU}}$0&qpGrnH|G4TG#K({EMJ^%vsh`&xkw1s*P#E(e5Q*m-h zD)s!cY|6!Fs?e2=k>YRfkvW)Spp!b=F^UQce-pLbm*A1_Q_E~s3n~8t){>=aS@Ofy z^544!V=X^yEd{EU`+wM4-u+Vg!V&cY(#5Z8S@y%$a-6D#qyGo&%knP-W0cwdX)W`d zQF+^uM&&3~3&j%;B?|I0ssCYhTJSF@MXD5E*O98+vP(L)@_*2=W1NoBfs9DGj^gH8 zRZE@KvCuW_7;Nf}21Ol$FT)UtZ3zvwUk8R497%budYcmYKg5Rrd$h4>r(l9|{Rgb& zSE?3H3_oBk2jbEfHM1nP`u-g&owfz%JGo?eGN|N&b8gVIfj4eRTTx})k zx6Xd);!D+G5&YO&sI6=uwglusabCk(Vaa6}O2)%@P=P4-(qbRr>EO+3fd1G%svdx!4&F|s@EV-|iPEM>lp~H={3<#W%!qUG zVKFab>|^wsp0Gl8E0iSxzFvyWxGiK)`S56Je-a9A)lPnKKo7reeZmfbaerP0%7#Bo zxHPB^gnNnV!7pK0`KU1!WI8~6=A-V7XqI70um_^*tau=tZXG@AbI_D~X8#$z0|N;@ z1$Bgoo{wYq%YKRNPS$}VpGD6h`K^#V?7`e2i_NI#;WqpG1SsW;s0&bn4h*t)5wznA zN8#{chn!(~W$YImvS!Ah?)KtlAgLAU^$T3ExVi8k7B|+pA&+91M(>Puq<=EL%5j#a zfyjb-4uJ40diWbXyvAL}r^uL0a~uxBOV@ECuna@^SU|@vB?gDiabn`e@oR^V}5TlRq4sQG4_HA{`Gc#UOw>&dr9d4EPWD4pV zya`@7c++J2b<%oGjHp_0G@E7?zmi!w7st893ai>Ep|r?E6yhz#W}2mYXJHL(&hDc< zIU8Fkn{mm*f4{=?eeu=byX|JsUBNB@X)94aKdO*EXAq8H4Z3HskF5hUu6`>ct#B>Z z>ssI}18E1d7rSSw%#B*$cqE~e5~~zWVycrTpOVzcA&!0JU$k8rkzu6m+gblB>cKAQ z88*AL?GSd!{z%wm7|oj9#$dl@1=QA8kVI+z3~%Y1@e}c*=U+Cf>^Kh&eDn-o z>SlPpAHh~LbaNglt|*Edo5om3T2xOR{@`(r{2Q&W}Z*?H+I>^F86Ea{ByyoI7$a1|xilcXQzKGe_GE zWKtawD(I}TwLju6r0&U9qYr{ZD+Ok9(Cv{dIOF?)zS2HJy%ls1LI>4~po}tsOw%mbXzFoYZ!1J>RfD=%^h3Ub~#2yA`?KL{ZD}$P)=r3?~r} z#^@kii$ZPlD`1R!FMeY3ctA+9g7g2%Fjq8^>k+8M!!GCXpqP0)>~dZ|eE(qn>xfR& zOJOcSxC*zYg9Vnr#0GCWN%aReqFt)y=D41Lv)z}!zwBdA`qw(dlDR?zL42uhD$zjx z^_Xjf7}&&(!T`cPXtfXGmxK0aHa{!%q!Y!0Rc`tCaLSh1U;o=%W?>>a_zy6-wTYni zpSW+giC}J<+C&h$6*q?Yj7T%=b}a|7Gye2CXS_MlKb;I`=u)>^=xd2YfKpI`p=RL{ zy}MCdZI}$o&!itMC)9|}@>-88BX!7kaMk!vFpFk9upc7}ui;zM0C81S;@oAJaS+!g zywndID0Mw!ampIEUCLIxEiH{Z=JnYM zsk_u&0Me5!XXvF{eWh=qf4!Qvqko^5)MFHY>re!t>j52fx8SKL{4En2Bq0y41GYh6 zTTo8|b|qkug&~0kN&fDXe4hCuW2cya2T^*q++gesuv0Q|b5W%4Lnt9_Y{c^rf;8Ur z_J7j=;>hshBd!NHH&x;ulc6vCNm2MvVc_G_3=Dfp2Yx%}ZD#hz*5ajN-u0%#Dfo<= zk{?^{^R&Cy>E}M9Mq<@r7F3|!y`doU&4NYGxJLlU6aHu5(-s}Cv>;QS5Nr611PU0? zM*1JtJG?gx?-@9(r2ppjd4Bf-Px#%y4h+<#vdiDVK&(W;+T9Ia`1J2ym-q6bV)uOe zjQz0@;~~Lm&`miaI4?lyUhHSQyWWelmhW!JYghznB~>n!Lk_$+a)>f*2GaDvN0pqO zDGwWW+}0 z*)Wb0!bLpk>EWUoc4-`er6@2u4prbNR~@a|2r$OM&d_ShaSO!*g73H`^oI{$A#%}Q zI?i0Plk(1M-b7dPP4vLuLk!~XnU?0yLh$Y_Ha}wHHd;>!q4wM0C09567~Q1(-K729 zrZgRzT;H^J@}{Ppleag0Jo!BcBUg9)7~KJuQ`{Y3IVIge=V&GAyo!akping2n-Aj% z?|Bf-{%t&e5}8|>fK9#ZOo#q79Yk%I1#8FwZ7-MMoWcX$DGSKHHrHy^+-N2Jci{2l zS?IDJJvF$)GroWz#d&J@Ed-||G)B@O(~G#deuHbteiM+3(vhU5x&i%6-z z2^X}JQd)IYQnDkzr9JpDYQU)3#Wi5m?2;O(2Q5SkhzneZ3tYY*n*N-ErVir~{lEC} zc1Xi`)M{&iFbqSg2v|X37*1P)sTF*k7=Iv7wkwUlMp_;(#c9M)c1fB-N(S?Cj11=G z6lX9mrzArL9(NZ?X)QzqBDV9lX+h4!&cNHZuDkzCd-SRC4$5Y`DoI~Y2&cZuaEq(q zrd)msuMR#_;Q8RGbg8#_!BeJs;ssAx>WLRT@$@Cs!wa5r)e|px>Z6``!Bb!LWDl^i zMTqHKkmk<;&MH#mg~H!*_7an^;4vkixO1iR*GMPMvWwG+v+R=eDNT@J>G(kgevyt} zBvXcQEFgM(vB)qRl_Iii83t5h%YIKKjwA=wvl*;C$)3jUU=ToqP&YH8|=^A99%4k_qm0pInkc) zDHJ`_WTEJ*tLllFjiQA$_<6fk7Zub4D*15Jl=@UwSvaX>f?zq}{E@t5~gL48E(-#fWPDxE6T&6Mh-Sxi2Z!{kFbNVeyJO(_Bn z5jhUMG52K!z(iRA3pZa-u$_@1%lN=|nOfjuG>(D>#tvxrc)%iVCD-`aQ-k(^&-xIF zc5m$I+@^iwBbaKQhDpK5R|_rwkCBw^dYZPgQH>dCv)42oytuvtSlK&+>jU5AX@Tn? zWWmwc9)lYS1-3y}20Bz_N{HC=$g$TiXwRZ|xi&chwpo$q>AsB1XN$5)fFRCUXiKze z#rljit(yDI5-nq{=V=!k9btB$8?#Z_k5HK^)1_cC33IK$ND+Y9|BellsbI1QQzS5l z(E*n8x&xD?U|^9Sm~pzm9JFAbbYQX-OfF%*TgUe7vtUdICRf4W&K-VvCuESPH? zm_7;ykt_oLlpXuTG*q{pzG*=+m!GD(S}IPBlA6)a)E4WA5pO@yEHRxoi{HkS#ze zzMP%);QCk%Ns$<@9re+QY>y}r*-DAX zR!T%Rl!&H7SD{gRuVSNiUd2Xzd=(l+`KewxlHSo=wwz;&4c1|Y%zq0X;|05?=pBO{ zy`#hY2+gDE;8j@0G+?MXTF58%JejKWKcCt^nIuJCXB#2Rm5E|7X{mV=Wvb~jgy|D$ zDH=)$vnbM1^pp@LQ>1Y+b+nZb=2WDm=qw>jtw>AJTtb*(k=9Os(L%*wEX)xQdyCy^ zU7yV}9jb^SNTAlTS za4j3=KQ+ORjSaJJKPnGB@5iAAi#&&0D<&6DC}ePzLC8$om<=`~;qWiTzrBU#y=AXyiQii70| ziR4`*Di)R}B$9cNsCZbOkVx)DqGDosLL%80iHeKm35n!iBq}xz+WMjv@(CNa%`fK( zg>A5}@q6gQAGGXiu$I5C>}xmDzIG$+Yd6xqcB8Vdc}tsp5qFRWC))ZEY--uA6^Qx6 z?i9`)hv3+iJuJJ*J}yfCQmiuZ-zNAc*{5K88W0wD7F1TPmLCWZO(Ow+RDZ&PS5y57 zSO44o1oX}I7Mu!2s6W+_P!?i{L_F9|zi;nQD1HHoB>G(}`y^652~cupf#;(73fq$~ z5F2Eh4wgi?s%zR^3>^pg5Of=x4#KgPEt=23ZCcHC_lnMy=*uq?qEJXsZv)6;@wDYm2ubE8gFpLT!0p$$|o3<6DTaE5B}QTL~Al z@}_#U=T?YY6~~LGSI+B;a}7Tq2z4&~d?9DI%$59 zrz|5r0Z{N&LKc31KP9kQhsIm}%C&D;{z_E=+>~l97@=`fH}7qc%d=6rtR*>$;0r^4 zXak@_GSrGRpK|TdAz4qK1`R9pUQ#X_mfJ7k?pq>-sj~wCtfoiDb6AOYlRSru0NlIq;F;N zcutJuYV1CRewX8VLe4_{lH++N^_oi5AHR>Xa}g0~RfQ-TKmDKfj~c7%i9|0A1il=hf2qz-Hb$#`0Kbq> z*uHiP^hs=b$bwoLAND=v)t2T|dG#M18H^Jz^-}-6;9%#_;5$TAg*iAX1(W6F7{eKL z_>vrDQyoI>L9HV=J2{l2e|n7SO`BG`P&$Y`AQnF*`(JJtXFh8AUuMcqMjR#%0g8h?W05HOsM+eka=F#irn-(Y+oN!( zba#J$@$&URJ&F0XVJ?OE?unQpa4S?y?Wxa1G zJCp%Kx+>66wkV?}%SjM>lu?uIBv1uG8M#h^*rkk`K2Czzri_}tPC`Q8xB>bG2nB>f z(iH&tg<;t-F@GXXuGu|tQ&V)}_NJDJ?>YU2EO*!7XQtSD{WU;hc16vU#vNDIz2CUw zvbw)F?(o%ZZro8+$K#aYRC{~&t4$wIf@5P692}GC8$X(NQbT%E>!gOJZ!56fzM>w3 zf)#u6cWaT~lcM+IEP4)}{o8eXBL}YDz^C2)C)xG{&Y6unZeE4YR=}9Lx$Fx)aIpLA z8MeNIaRqW5geBSqsK+-p?znc<%Xo%!0z`Pc0k^=WWJzjvA~n}eEw1T<*B;*{0DTFt z4t;?mqM>Y=Afiuj#k~9r9B({2jy?JNwaDW{ZfSF8*zN1km&}xkVAxZE;F~P` zTls)W!7!6;6f1sII9BNj$tqo8nfErD6-+9g@3-+hyBp0F@iUvCrMqVlt^c(V)-BY= zFc*bf3;PtWoZ0CcqcmxPsr52^h8c?fuP{2VBJCHuX8-~dDn_faV2KhHICK^~2M7sa zr)q;0}o(Q$XJ4&qiU3dj$=7jPbO3%NnJrPkE=vOWDt5)=@R`jbr>KOUorC)XO z0O^E&XdlpDoisv(2Gwea@ZP9SqF;3q{i>7bSFMJLdiUt`JB^g9+>jU$7R{1r_4t#R z4q@*ljydUJeVZ{))c2eaH4X;e=%sJ@N;eBpB&3g%j?lx20AvdNuma-;u~8gWymCWQ z+9_`X1`IRR&T!&l`XB|y%0al`e6?2rW*=&TGfmnyA#E$tio&~n5KS;{NSO63D0 zVAM%Oh(b-wefCR;UWw`bw-JVGgp?u?P6x|}C)^PDP@;5pqeKMv79o;`cc%f}0 z61G|iv+$N4(WkKb-BMaTc5kVe6uv;J+WFMeA!Q|h2GV_zQBzc0%rdEau?p$I+fhJ5 zsnBY#K{d*6h4Zpxx{jAAf0f{zxW}uh_6?5ABirYW;;>p*hOcrcD$!x3bOks7Utqf( zEdh1j&jfB**o|ZUsoAhGDe_h9aS4ajPDfdoeKYhIh_(=!R_t}Scj4Y+IjCr1!-H~A z(X@sKQ3zm(kOxr)V1bYaQ3POlkOxr$U~!NKTLCCr0{qQS^E>#PVd0N>v;5ug@5zB% z2t^?B5R1SIf)Q{zx{arS@hF|3?8#)5(Uoy3#|iO9e-re7%zsISoh1B}r{oK+FwXkCQ2qzC*ra@aA_Vy3!V_i zIo;Nhn%o!7sSNm3(T&|_D|{j!ST6x4*WLU;E?V^KRh%4<22B*NK44&Gz*8~s z35a6M4U(6w@&up7z^CL1)0F_9c&`LAgh2L1nJ7ir4xxC!XY0V8e}5b{rJCNS0gqoz zUTTSPgiX?Xi^agw#w!8lfr2=vkvr{VILGe1J0v;5rQ!3y=aXO^} z9+K_o-u(KqB}f%U7pE)uqyT73n?H@l^h(^Nk)O#TWy_Pj(hv(|VJ1>YDCaa2!O%>k z$WT6NCQea_B0%7=wwe7=5Bz_uPm4T-nsjn7S#>v(upF#*(m>m6h<-;tp*?672N5n) zQ70}l?OM`U8Qs&k&^?XI_i|k~lszgRdsLx(jH9#agMN(u-}ZQl7n@A|G1uo6XeF#Y zI4VfldvFLoj#zf$BcJ)0_S+-bd-T#KZ=@td{nX}<_V$nw*p6@8Lp0+Hlco>D{Ik_b2p^9MyjN(sQZykW2nE`z`j$E={tAEEa2s zsqop}Ci3W?luhJO*&5ctUMt3s`+tLP0PULZ2bySDOZgsPvOVO|i_8^b4>1sbkK-8U zW%DPGKaPEkf`TRqe6UdB@3F5!%pdM=l#5tg<&3`nv?zUV{E;O|R*!UmU|k1YKv_5J zp@XaPL2>1{Sj@mp7-EysR*5ksaxmD797dUDb8ys5#4m=$ zN?fo`;dy2<4{e`+Rq8zCjE^61{`pFaB<7#qXv+Uz<{ui_IjvwK;yS-)!5=fyZoL_I zORXT^fm^kj3B3fs-aVoWj#`Zu$DS7Rs=3%vX|=#P*lZZP zmpcwqkTCm3N1iH4nq3TMoMW!IUW~17Wv>=7sEs>tIAcd2Sb{q9>VnIsbd>;s% zQThmk%UmcF8$q+#-DtUiWdSSGPoL+?)LLSgmQ_KGKU!{Vv{=;oReNf+qt_2%zX6$O zzY#L!2_}=~qa=gaCY)E6(GJtMq_j`$JHYM_X1^i2?){PG{l%*h|dF0KB1mZgd4% zJ~t9_C7>4!Kw%O=TCxt@a@D%4_f+&L&iDmNN5*0|Z2(eY%||ETZf;I)6(lZg0crCu z0EF0p8XL+7uDdlCx%cKFr#Zy$ee*B=BUA3_k7F8y1Ygx6zd>OT99%D>kv);VjOn$f zu-T=dL3n_^2xI*w>WQ=X7|Fc@Dy}^@Xtf(4vy^#!J!;a!ZyNo-ur|VI)k@)fi%y^= z$Tj*m^QdeX3EC?8h<#?TN8inq$Pl{z)AX*-$if^S(F10dph5~7&q%o#BlGxFJhFh7 zl95wzDRB)$q(q4^3|a8^4?`CG{llg-9m?@sunfP(1@-tXzMuuaeJ{`hUm{-pK-uG* zHd~~7hou=t{+6<=*vUwNvxe=`aWv-sw~X{H!r?>N1uh7sS-67PgKTh`jO}N6DRKP` zGS>AJj>Yn}Z7lmVo@oQ?fwu0l3ec)$YDeA61(zf4T+;r1&+{LvB(8T#Jb!bcOwaRC zl}7a%X|m{key2*KmW?!7dq2NhrBT^Nnk>Pe->cH7b0bYw;?K9LG^*Z6lLh(nhg2Fh zaHMfv?s{HF6r2}^@X%$L8z@WrNxO&CDw4kh1OQP+e|*GINUBw@K#~`>sQxX?O!HQ? zR^noj`Zu`MBO@$QuaP~0bb;Emu9CGqez~^Cg=>3UxVEnXsa=g#AhiowAhiowAT`21 z9>G$m`!ZgP?uZkFV&A=mm%iP-2(R*C^#8YnSE1qVSs=558X@nCio7o=@~#o`zNpCik|OUKA@7Tdye}#8t`YLSsL1=0 zBJUa@?~97OFDde_5%RvM2bw|NRxNZn$cqk8zoQ{bMdPZWh@n}_(bB4Dcc+|Rm82bj zekuKZ1WTS8QUeYHQUeYHQX_jrP2(S~tSec#tnm+*)fF$SZ~TL=&RE#e_=lo8*FrDs z3qH7J8j`gHt3h#u6HXK!$wu=!QyF9j7OikvWG8-wgd-B%I-=Gdy)_`U|_Tgz;4Oet8(^APOHjk#X~U$Cf+H2>JT1WXzH~Ops6x0_mtg* z1_;sG=EKCW)FM*u9mxuT7=fJgM4Jv}&wL7kG28zX1Y>r^Q!PyxbWhP$_6c2O|D>zz zEt-8`T-^YF*+1bgOEUU^8X*1=Og}WF3HHxpWBRG#AXwsHqK$GEI+6PuV&r9ha;a@g z*%wp~kl~?jk-*?gBokOzrsfs@L&s+(leIPG4H_UBIOBj~RCnk5($M zs8UPr0uF!u_`873x{LX*;Jduw{B32+I0Lt!aU9!K=ChooN!8^@%HJQ}qZwP;>K}kM zxgjih;d?nLinaGv+`xcC>$GLCtZ>hW&!WK+IwvxfJ_%27uWRn3mN0W-Ul=ZDPMiwi zhRaL{H(ZJ#+!`RPy6|^u7i1w`y$iA+$GX54p*APfu~0Q(!R3vL^!&qG@M~19!qJQ*O=GduH~T*sJ9?YzCBvYO%a+Dchk34r8Acto~Xx`;oBE zVfW}Me*-vHaIdRZ@JkiVPOuDe#c&NN(n8Pj8A&dc7%L zv7P^7Xr)A!(RLw;J_{%@BCZ;H>LQoSQugbn5)%S|T47af?4Lcj)`unv@k_0sL^^xJ3|MsDtX zGQs1avEjO_a+gSw;`yETWY9Q4At`;JAfG>r*2H&{1ZFi4^shu#8f4!O;45(*mO2KH?b;6$FN_ThKtYPP|@nzfEt-M9k{UbT#unXBq~@vmy( zWyGqcLpjL7FEa3pbo`E3#kOJYFZ-*3&<0?L?zl9ooj6GooFRiM_`)GJca0^@T{RYn zEnk`rX7^nJ!ad&iZcsR<@7)mV6s(`cs_`Pd++w+fSHeS^EeM2NL6w48x$sGFDi=Np zHs#`wU7PAg9m9oCH>cf5^R^*>Nl{NS$50Ij* zrbrgVuc~~2)W`4fA-;L5gB-uwA31k2?5*gk_%FD&+{sgNt0`ebElwN<>NgRzd`rvOy@%UMymXwNIru6$-?l3g7rU^KQmB8XY$8T@~4>c=e7s3Fa)7+ zDf#1}{DH=${_!l? zz(NBjsR5$}40fYQ3zXE*CIXs;jhw}^Tr^6ss4+o{N(CYD0ZlNFL|B(qYSC(IEmYfw zKDAX_FL>=P>_P&95H5m-%sG2WLh#$y^L_pP`SBuq=JJ_2pP4iB zxy{UHn1&$>L0}bTF5p0Kn1=DoG|c0q6g0MMCFmokN$2$D@lNQ?^F$#E6-jDzIzs_D zZ10o!;diQ_H=ub450-Bgzf(U*BMQ?#$m3(i2Soyv5BSS0JeY+CvJ9Go^X(mIj#9lv zr!=$hnN@HyeLNDKD10Eof!@sfYoeexi4l79oG30$7}$yh)8}$jv`(xZK~lRJc9CVm z_)Y2U*@6aG0pHyRr_>)W5W1~2riS|1K3<48e2slvg{P48W`Z;tn*bNG=N5b}l{C+|U5W{Aj~cU=f8s*7N$H3$;FJ?Aktg?Aks>feXpzUii^^Avw9A z7m|~w_CkjRu@~^OGE^jFz835S^oh4Js;}~&RbA(}9Z;$MF%YGoC=rtG97Oj;y&p;h z_M=5s`>|h*XYHW_&wdE>8T4GwN65|-^<3;1VWF4ubUo6AKFX8z2n#)wr|JpizDZ2vJsTqsY{5;>ZZ3KBV* zlBOmWaZs5R{%Q)dk_!}OB~MlKep?UvJpvU?0<~i#&{hm>@6m6b52KyphK* z5Yxq6tVeUVdnxvwd`)Zc&ga2`J$>)SVhvKTzxB!&+uABC2E>)7^LT+50}LCK3Hc8R zIR=nOp*On2zvYHX`i`S>w}|gJK*P~f>`)1h1f}W>h(>TS4(HiEuEsP7p2p;|eO$?4 zPz3Q#Y*6Et{K-sw1V3;s_Lo)NF#vOLK;kth55V~yJm86L&wKDh7l8R*T+Ue;hMF&- zo|RC4Ce#Oj!Y(#6=EN)T`10lNV*PQ)+;5a&b8WJDZn*{w*a23ZV&x^Q+yvxsU2qS! z!bo+ExY0v-<}C1ax&pqI1iR}5uZm1_fkfD?sc>wU%s^7sZo%(7W3H*$tvJ>I68n-( zUTjJBYNl3)XXMLv_$3^I%C&gj6oQeOiuqzw-kos5wZ3Yf;%MVeYhTcez($dN@)GH% zE|Grv66t3yk=}HP^yW*Xw_GCq>?P9AT_XMbCDJckBK_he(pxW)-qv}#(n+$TYTftM zwpyiz7YwQKQ>|(ANR}I-QfpC`3`J__Hb==<_>A25%$4j|eq!_FUz1;R#HKhnIPl3m zW-n*H1$Hl+xEJSSV-5R_8Cy z%Gv1mWj%VrTs-GvOZuk6fq6T|cSCu5kFPHXWOUI?`QqOJHODsJ=dv*lb2 z_#SOG7oS2Mt%n0=cg*hv6VdmjiDEUjmq47wDhe>e5?v2-Ag5q+z&jZ?B+eb$qCAiN z)z8%xPrAvKec*{`B>J26&_e}DMViv#c+IugNm)J#KQWtqCUF4;9 zk=MVAytFRztXi@af7)QuA{)etB>#n#zJdblUzT$gYf^Q*2F(5|1{(rzVXzM@ z{r~xEM4tHnhQFr#Z}ZnmgTF5LuKX3l1ZKxh7;Ijg;IGrcd362?rkapi3BC#Dnu4cb zvwiz81;>VvnqL2;WV1s)Mu6<~eYbdy^5Gf$0&F%~-}H~JNL|k#FV;y?df;bj575TJ zQ#bo<*;DHMHtW=#4DAjKF5FJ0TQ8Vyy}vj!XGg$VxQA)GMcQtE@t~Zq{Jueto&qEO zyCwZ=GU9v8sIvf!m^FhcMc3?Q&0f^Ju^l_#1cn`8S`^z~>-Xh8dK@*sV@Y4fnr}%2 zdrk?n=P{UB(d_xgyEg}Jyw~Tq{q$}$H^o+g+FL12Zh4;itY@A%In%1bMJ>rsj*imsW z%27LRDzFg=st8!Y3FEJNSR@fed~U6GH#l35J%8$7UDwyxLv+otvW+~Er**tRs>{NZNxqpE2w6k#?UPQ76eI4J6Ln7-Cb z&DsbkvE?`q0aYhTlRL;k3gd1`_B7)$&YRa z63M$8iNq2A=)PkQ7vG+ck$$w*Kc*u0s0n$Yqkh}GS!eKT1n&?*%;iVAfeG{rGXeM- zBojdIJ)TtF1DGUQ3cP+>(&QtY?nlFFS|3+ZQ}K%V3!tp+HCqa5i{F|bx4I6BTA!Xt z&7!QOc*SFJ_g*qPvFq%luCr6R&Q9$*yMNc&XpFX4*VzSKXBT#zJ++Bg_XV2<7yR_@K-4gMV=!O@NN+*p2Ya0aqc!N|*W zMvf@clB&+a!m5#vE69-+Ru>M*gNsQ{4zR8``O;Ya?Q4h!|ITa2yj|-X)1!EIjJ^uq zo#)mzng!z~fB1jQuT#OV%dn*>hF6n|#_;Ov$g4HAWLGq=7WTk){F?U0Jk?VgpvfBT z93>gGs4$vQ<4{K|qef4$ByEn-S6wa@{5e^-H&(4pqRo*yk$H~6qFZN!%fj`4ccf4! zD&9NalGOVnq>@<=`EO=JD|&Ulc#irTpJ38!_ym*QP%55%_si#i^YWQ~MH!g%M0_TG zo``(qUtN)UP%`auvYm3UojkuSaq5{s;uLUh^VExA;b)>)c)&d60yy}^C=Px}=6wrH z8Q+z23tlbWpH{rLJR`~8BVN89?<<~Eo;LMJySE0GO*w$QIo||Ye{nzt`=YLXJ(`R36;fSVkj}FNI11 zN^en~`4Iqw#s;skTu58v@Z=?s%%{>t+;y?@pSC<~?QN#&0iA{wcz%WoI?zEZX8ZOg zsGh+#hbKGZR?oPjjnyX8T44}d-#C_BfPE!;O|CmRbI+8oZny3P@^2J3+d8a1RiYMYA3F##Li`g?@Pea9tQ#!XI?Bm z<;+OG&;tLe&3tteU)?0=^t)FB5Onm~Fdh9C!zgebB(yqETW}n+#@u=)u>Rg=G`YOT zl7P3MB^+BC6I&J&dod>Va!hP_OzgFo*vgpL>oKu6V`6W|#8$<`K8lHXV`Bc8SZz$~ z)0i06lA zLF^#P?w{B9rnrFb801z-=N`mfg}^>gxgrMI-FGeNUBT^i7TDHDS(FXah34AeaP5HS zQcqteq#d=yyihf5Zfmf?&g?8{@tEZSl@}eRwX}nHvY*0c3e9z{apn`aq5LF-E48E! zC!wCfu7L5lQ_Svyhp-k)c^bk_*B1YZ`7_PW`CqLVV4jS7>#@HDUxhW_RWJO(u6SGk&(nd;5t#D*`M7Gra}k3XiTSQI`I%dtxaPdM-eFqpsM_D7 zbO83RtiX!v)8qr*IRC5j`kE)iIof=Vwhd+9IN&EqS;!Mt4o|`OKB`B123s`&Z+9Jt z6tIsw87C7PqZ3c-iHD;ToAgA^ApL#zaV>h{>FC6BdSXj-;srhNLUdxQNHn`5Y36b0 zH6OfhV_9GLA2oULRf3Ph$lv6>KAfYo9zMa{#4)$$VBYqi|7O3kf^3%#Qn_fGy4 z4fEitN1hJAmLOk?1uXJmZ-FtBFY#%9yW1Sc;yei1p*Xj8_!`WYsp7SZvWhlvnhsSw z=zWv3P$wssK;ox~2?B}zo!3x*`5KmI$QsslR>Q;IM=pcFTZhdsvWp?H7AIcd{6Uv4 zAirQMOp7s>NMfx1XI>wCRc%>ohi^sU1Kd%Zrdz%H&ERQPEv0UiMSbr|qj3AkJW z`Usdyz#Apt^f2H967aty;NJ*%GXaw&;LTycQ4(;x1bm)=`2=i4Z{gMb!hk6fFkJ#Z zNx)kPD2_0?6OR1aXiR9PXv_sX(3l4acpCx5*+qAPKMc590&bOnc?7%%fFbNcPk1E^ z^|FNeyM(%$P}2xC4p4K$P>)HdMH1>X?A`A_L?|nuZVN+Alu-9cs7-__1Qf2YUHf0y zD)-i$*0yN*e!<~Y95b$MklnMfCGX(egt*4TWw`qQgra&)J5?ae$7J2py!hkHWnwuy=iK^ z?$Oq^tljJ4`C@h57;eA<@Dt|1*~#9(+2k=v5lpdzwDC${s#dj_m6{T4as;3ylFt>R zJOZ73%mBIxK$_>_#0U^>xg^MEVUQoN9#?>O3Rd#nuBlj-@r+5Lnp0?V=VPN^Lp*3} zWhcFXJAE4iXh4U)_y()c>UJwGlAXUBqF-&(Tqm7B-SD5&U zAg>TakA&CzL}@yok18z&yePd1+X#Wbfnaa}07VJfJ)=qhE&Y8jhf8!5@0(zh;045@ z#AtMKDm$6IoyI^^r(k_AjU9IfsxkU|+Nsp24!;#2o*kaX4p(Xp8|A~D9{dnVK(ci9 z_(+GaNxVK9x38l9y^Uu;8}x>ijl-r%a}*8$1H{${1RFBmGvib$zXXg?Etc7t_))6= zh+3<>&G0`s=b_-_o_K9mTP)z5&-2~)oAJq(JqY@#ykUi7(u0k^1Z8lz@-4)2sCcu) zUVgGE^351De6cwG7WKVw$umEAcZ^el=iq{44ZSu|Ou}4*IsO;+<$mP`pl5U64O{Tn zD%NTnk7KOzwYWvR^aDDuZ_-?FjRUCA4-9)ZS^$5-s0`x8nj}Z=+Br*!Wuw!5zXjrb zf7?29e-wywth4uTKUIv6_o8SXCyUK(77hb>lE*VwpE9mThz5#_DHCbGTW-*YV!;t` zJ1P-Vr$zKaFMYig_|e${kq+KY(FgN6&mE^d+>mn#(lhQ?5z_M`lHsACFb{!Qga!-J z!(Pl#UAEM!d6u}cTOF>$!FVyv`rb_-(xV?D;P;*SkswFc-Yv*c0n$K@9>W9VsODjk zqnN5n-&qB>5g?Uz>%^=&VPSYB`Trmn1C=)v%LD;!*qsik)~S)MBa6LHBwKkO_HWk0Yq zdR`rr<9lAaO0iQs05z`4Z+7HvpVMchscb#mXCIRNQ;n1Ef}+=A3KPSO=o-&94rXP7 z)%`aCTyW4;K5&?XVNa2bStN`_si_h$euwFZ({$LFKKc++tMH+6G~7fF-*5}3asM<1 zbcVVvKP?Rvx$_f0YXm172)c;~{eYd~ zg^^dNnOhyEFGo_ZX#c{B;r6RjU~I~T$LO=CeSx#HYL6D5ylBaTOQ5s!y<^?wS7(ebN&FC%YCUV& zo>FXc@v1L5n$NN=KMc+UpC$rfegxYAwz=T_*u_#a3Aqz+!$fPO0*GDcIRh$4KOA=U zj_4ev=4*Vm;1T>pTIj4fpj10Bqakx;SME2_JE*UrI~JNM+hEK{bivNrT{WGk-mNP6 zwJJ_eFGu54_%mFIC;u&$4u^!}8$>yyak^(F4pbo)IQOL43KgL7xka340f);l% zy}PVjb>|NwbP;R~)*=X1QaZ#J1*h;frY*zRrR@1K=?xseyI0jB3*OM=00t$LGiEI2 zu-F$fM*!}m_=RmSRfJYu`8l?=`N<|RjH`0u%B~5H#~VP6@aEVorJC+;D12t9?lEx2 zR53g+CD8gLFmz8$a?gTDC-DS!Y|@jz0fM*#9k8l<7LEXXycutl(rx_o*0!}ZT;-&X z*X+2mG$H`3_>lr5eE;TN`Nm3ga1>4=SlEH2_A9+W_lRLEMfW+26qcI ze*2K9H27aY4S+X?0c{fS6jUOB{TpE1EtS@IKi`f}MhHhP6CgeShn!p8+ST}|!;591 zPedWMYf-q|5R?ny>Z*hb!s90k{HqX- zIthaMmW1njT8Q_3e~$+g%;~81{aYPEi;F|ujiLLAdBW5%{9O|MVF{l@cvZr0#^-4K z5b#m-1h>E7Bqd&Nn{2z&c89ItcGY!Wbw8OTGFBs_>WRcSrTSk8(dlBU!~LYC@mq`& zh(Yc#iI9Mls^^fYxt~ay;IY1YtGoOlC~RA4LZcT@W8C(GDG&zBQkjv)jI|RyGY+OS z{)uA^no(6t&$54oGygilQ+_bjR&}$b>?NCftQjog*S3!=;_XL5SMa`W80xD05WO}k zRWtE@9qtL1aqf|WSoPjUP1HKp9NK|-@9HWvW|J(Q%lx^ zAD97av9!aF<(ZE~67yo@RpO(y!@oa1O1M4ByKK;<-*?&k_byZZGWnuEqwoRV;R>Vs z(`f(Y@Dc5A-yZi^(WTM#rSf||TnAQQw+w3!4}7BYJA98-WD9E|-}`?%p5+x7h4F14 zZ~wmb`TyJT%e_w=oSy%H%>{-$*_)qL7wyW3`+HWDCt05s?xmhi6Yiy+&f}x}=_2}< z>hBWnrCe#ky_74D?xpN5_*?N|5!x5)UrIWb>ge168*=3>@Gq(*;H_Rb26LEN9Imqt zIKwX;6g!_V$lwlw{-YLP0hpe_jyT{MZ1!V@KN02wNZf7!=L8Gr`$$`0=HTT02JkZ*^iryJ7bT|7A1p`YVRM{1tFGk0~D+leCc!>+v7e>5+VOZ**! zHvfXzQxbm{IC)oH;C!J^{{&y)6U0leDqyjhti{mxQ7BQCU(BE^znJ9_)kb$?)hW!< zYzRLzC1o+|4IH{_y~u|PI_5J?z##t6l*CB+?os6tjxI0otMZz0->1AkK5+gnZ-?)t zE;vjfOl;%oq0m2wuQ6f4Iri0@9@tV~!%ksdsL-N$7SS({?~u7_A2xlb)MHIN;U5#A za9^C+do7-jcCYGN<3z#*I6hboSj;AL>eV~D%e+U8yh`MWMgMrDFfgQ5)63r>VG688 zqPl@bG*3itHlk@Fnj@nrRuTKDk=Y=kKQ^LX5w#l8H0lgOq5-G%z<^VFVE9P{9#}e7dGBS*qYxut*?8o5^XUlHWc4h zk@=kY1AkRkTDhghf`P**(cw3rg`D=V2Fz0xJKRyeQ56yfQST-gE-kEa(iJ6Fe z-CaIU^x*#T`oNIgUk1`!acrsl7rXtokvQ0d*B1Y@BJ-kl? zahW=@W;5#(!sE!u$SL*MVmA%L#g={Nm*Ju*uLSzv`85mG%R;O#EC0q-jLtv4S8K}$R~P3HtP-9r#IyEWg9ZX+VG#XVw%2qgHE}ln(?&WjFeSyy2E{KyT^+mXp-heMUZ@?3=4e;m<_@mx{xAX?A(HpQ`Z@@vl0cXEM z1NMheiE6<9%QYaHKZquO*?9xXV;k^{-hkzL1OBNuz^^x8hu(ljy#eRGLj&3~2Z7Ev zmuo=u@E0Y%>bwCJu?=`uZ@>z@0sqn)5YQX&x!!=F-hlJpp#kk@fIz1!{#KmaehwsG zdJfEwZNO5!0e{jP@V4H7wR!_~>J2!gH{il$8bDM!%mJd)XBJ0+tbG$2d-P`Dceq zu|J|?-}W7_?=bp_OIM>mkR3C00`|@a&Q#07D9o4XnEyq`e3g#*dL8pGbK*>}ba7JFyIBqC}XlJ#mJy(vs+&CdkwYkt&{ZFJoUF81N$!aE+hH$~3N6W&J&G5MOLNhE(IDeDW1A8|i z5Nq#H-n$XXNY-uL0!Mc541CkND{ujN0yc$PKM8zEWg&UW^1u;l2}fGDAg>;&^+=sW zEimi0UeU#j0^ZIFBxB1Td<((<9h~XAU{j}BEKD0jC~VIP{{G#zX!|#-`S(0Oaib6^s3eE0rwdhm8b`x zh3Th8>e}sinPr#(t~8=~qQ)E}nkJ%IGOC*aer#lN09@7AYDB#vI>3l77twx3RG0yV zD2NJsKp#CYsy71C9`HRQ)Vrq<>f6Hz4M;FT>G4Krh{*^IZ$rC8Eu$_nBWiaBiI8%{soi#uUn`( z!Xz-sKf~gJNr2Bkz$C!uTQCXm8AuQXOZ0-L{0kiZw=L9h^n$x!D{%NrEpAl6=O6V7 zj`SeKkZJ!yWz1Wb02FVy7}#>`h)me67lxssxZ;MjD6zS?9#)0H9|Y=-H5ETs@uTOm z7Yb$lejS1HG&6i;p=7UH96a5mGo&iPY^G~4Ye7*~c&gXhibUwcVMdwo* z8|r}fHfbpkc+=pKM*~L~jg(h(!xl(8L&p}pCaeqpX(Reuzk3^|$F_mOz21hUdK+HT+ptP+!>4*1X!2njq=}(p z8>U$s{?i6bwR*pM3%EcYHI69V>n-@T-hw~tEr3N&V7yUp0j)i3fwV4k)`I2#c?&AP zdkY?lZ2`r5y#>$dE%=Mx0=Q8Y7}M-V%xUJ~QyLaJwm|N(`Og}W(qSSCJ7dR*%rgK{ zAK=Ld`oGc9he=hSDNUZ5h2}P*P4f<)(x%WJeONF$ox&ZZ+NddP?tt~@os*v#(O7Gi zO1;OI=_s$%QKq4jDAKUTCrvqgN<%_Bl!clXZOM>yr#Ne5XZDN;)}kEbGu^bC_<|_0 zrE|o86^r-+9dVei1!8~I5vOUBh|-kCC+#_i&&Jd5*+M*Eeiqwl9-`*baOAK)CgNQ;L z3kCuqTJ?z-jmdn{c*!R%W_;3Q!>2SDbcT4e!WM;iR|8xi9?fCJP`ShypB0OEwT?KA z`9zGyV?JrF{JR&Jgcpj1!G6$2bv>8RIN*i7_r$v2(Q~Jox#)lM#Bhs(`PN<)(dOEYFGo7L$MRqfX57Lz^h1svgpF!Z`)Y|PQI^O1C9)Z24k$B>-zsVGqmH; z-2!KJZ-y?BiuJ!g!36LWRNi{tI+4{og!e9yWB)P!Og3Z-nBX zLdE|tA&c0`qTi>1i_;qmagWx#ge~$4^>h8U1Ww{A^(Zp{?Ga)lz*QzH_QHQOwz+zG zPlf|^s`oelM7Pu2Hbh!8#-lMnp>RMh5*SDyDRYJt$mR~8A(|WMKeV?8zum|MP|FI( zcJmjH%C9A7IAMmC13hb0K73hLWZm1ncusmo`i7Rk@YYZLzVz8!;c0S6M*6N6*tUc+ zR2Rse@>!rfIk7vr^?E&~b~r#A8Nw5D8I3L;C;G- zS(BgBVhYTA#C~J9;@iqHJl?>l(CWab(ly1mf1Hu-Rf^{X;5Xp_EJwNi(W5N2{?+*w zTvs;S+!hF&xrn&|huh9o>vYReKMtUp?_xQU{(mg(#>@3=5XOt5kBZ*c{#Z76AQ~mc zOHY}QJ_HGX9Fi|uYmZ}lF(@!VvKO=)D(0x65*a3zK>F_e#dC&dr0;FvFxqP7_xsXc zoWJrb437R7Mi@19ExJUPPc1%KA0(f$nt&J}N!W+5TU4`4RHF|NS#Aw$TZ?jbIXZli zIEP2reNN5I#_))Y_u%eWd!e8g+%GG~h6OYOM;y#g#?Xv|Z%djnL5_t1zvOezWAyGb zWysa&ItGScCHrvHUk69gOF3RlIHWH&6@i{0oQMIF^}g*w;PIK-jkr1@^HJJ4hUb7N z%zLEZMsx9Pzsm4nD@JH*U`XjUbj5afb#kNEtrX)QcXhtR zNgna1|3iIzSFPh}v8&@KV}pd*6x(u3`1`ZGB4&^xofQiXn{oe~a?o)LvhT zU2VT(xF^Ec)%H8~c|AQ{Ozrg*AJ$&{ozWBbN8WEa#qh|Uw26a=I?cNSXU<@0fC~k` zTDenH3Bx+(gshU`DP~b6q=~yNewffLoLpu<3SuB(SfSt=waVKKyA17)TrW=gqJg;b z&{R4ZX9+QDIYeP29SMix2|q_Ocs)zvDkPY2Kl2n^K?Q{C0iHpfNJ#s1)PR7?T%&(H6L))gN%Se&)B0YV~0N;an2NL#r36 zBTgpyZMjHg5Z*vV>Rimi53opcF8Q7dfjtlJ!B{>R_@?Abbj8nhV=NEESOy8 zXdgxc ztg??s$!Fg)Khzo8X+4mBJ#vCc_&}1raTqk|{);p?a;49qPlXpFNF#4|@M6@)Yc3ik z3}cF{@5w=@)e%Ydl=5(3cU0#UO9kp-~gC_*vlNR=A+87{m+p1~nnp6bQ67 z0X(Rsp$9JikVe@L4P==a5Z=1ide zUO<9sEuoJjgvxs?6=`0?jJ#zc&xpMkhJ3jLA}KYr~MAc7WU*1Gy~>xw8Z0mobnHVaNj=ARA*K4~HRTKSuZG=@`hCFyw`H zb;5HZKMFg({+=K|DBeH02`-S1lI~xJ% z)WQFruewh9_nmKoTa(5_^+E5XH0Xf16)DsAK~~}Xi#RVB#V>J3)Eby7af^LZ=>#pe zsdOCgR$HqoZ=6(pM~6Jv#L}6^RWRvoIJWfX@y4;KS0M#KUU(Z1P~;g?aBOMRP2!l- zRl{*?$(+R_eoadCMoe99VZ>p_Z)HT%~I)4)^%EI7NwGii>OWy5d!w5&DJgo;B{<&3Zi#gzHH` zJ#AQNmh}{|p4-i$B6I-uc%%Fn97LBDpgZq$8r@j{K?dDG)k$4QivgN+&z9`TV2)JQOcqZ%}DU{ zec^h%wIuOr)wHE;uj+miO$6NP!eD>}^tgMe_&_dSVepf&ohJYCmz^djJ<)0MS)8Qn zr0!x!rJW>i_@1N7whTH#Yv` z+DJUAXHIX8BomE-&tv1;jz!{8#fISMrt!9@agEjExVT5*wcy6_09AS#d1b_8GD9GfUdV{}LPj z^Xrj#RJEVH(P{F*y^&-#;`-yU@&7(;igT+4RcE^?Rb@DWjI*{ibvEUBuhDY-6-K{c zW%kad+#)Nyl9HskQ8hBpFq8svJw=)8X+d zybh@0jzOT+HnD(jNO*6oL7v}%j1PLb77vi;vDF~A+j9lC`!VjIYwY(UL7o?d;cF!P z3le_(ZGb;>qkum@1n}Qx3i!!k_&X&00}_7Y2*AH6;WrTeT?v1E7(QLX|4hRFf$*%$DZop4ILaUFg2TMSS9NVw-Q$PZ-F@IY;gf3<;wHHJZ-Puf2SV`4w@aycU5twH zIPZ8%6663ol0AL1A#YD8F+UCNp!J03xkG8CiFB+0$-q~d7c*&LJ#D%03NdI1ypq?zQEe%X>cfjbs#1L%tPL)0O-jw}H*;P- z0UsVX?+@nzszZ5W9gYmAsqWk1Aw{#|B>yt280X3x>rr*Bh9+z7J5$jH0Nn{7yEP5s z5dAeEn1l+BYjl3D!EA!(EEHPJm(N9DE%)Em^ubGwDYO%)(1BAPj*h}RlY1`uRL!GH z1|i_V~c6CJ-61TIC0%dwRM&Fl&IPcmoh9%Y^H1QX@X`7zgco$h*EPDwZVBH zdhGy0&fIfK&29wL5vB0xTeF$LD%>6)r#ur+(E;H!gqxwbjaVjLqj(M>f-hAL3`!_gwoF`eU9~r!T{9h{3LUtEH=!>Fga(b@^|qC=8IoJs zVWO_)UHocEumsc~3q!fuT8ZblsJIt#_!4dcNf$jIx;u7%BVUn*Y2bjEi{Np9^P*&) zzaw7g&mvyxn=0a;@di~ITXEA0p{ZuVN0o+4XJ;wZ=Rnvspup;gipa%tC+h_Iq9C3v zBCgko{x4+f#S1Zl=4RE0)EIeunYES$aFYz)D8`A&j`0P|CaNZS1~XO5Dosn|t8kkP zhQkj*(OI!5jLsLGhYPCNadD*&@N*Z^Eydid-Sn@Ns^JDrj7xdyC}KZ#xIbnM8n#q& z%z{43=(iefN1bpT$X441EqrmsAbU07Y!Q%DWV_^Ai9d<}GEU}CMd@%xOjs;Ys0nrtDbzfP@CrFs_;!2cO z<6pLV@pM%AFDSzP9b6AyuDMIXR4xxh${$VTKte#V&>=dByAz-WiF+)=m_$z3W>U5^ zlwJ+*xg5X^d?BVK{x++*Ta=n&uo#^M_r;PO*-6;F!rfF$c`Yl(%q^hBb~K)x0N;%W zBMdyM#poab!}xm`#O?_|j{y)|6(Il!LF{GS*3YKkIpF6wwlc*KkXWYB^{@#OguBLPZ_v7YU0_zZwC_ zX>*)U5-FX2NqPqQwM-{kf|^LG2Kprlu~PkWD94;xl2lbovWIUlmy|@(vt@E_yFSuM zmUL>y;UV6M=W=3OFJ?S@KQ%Pz()7$L@RU?uCuuq}1XbHCGBvy%Jf#v5hwF$^-HRV5 zOxW-R_@#CtH?Fjp*JCe5h_iY*p709=!jly9e!*C!BO+6?KEpA+=*98LK{`FdOm*Ns zEsVH4(lfp!^9#`!zKowXUIQ)*hY}Z1l2u57-zwGfxS(SA-adTe&&ZuX#rV;cH*jsc zAYqS3NEkVto{@&K5&8vwA~QZRh#XjCwL!?Xka*8!TSX6HmRiE=^{PR?J_pHmxX!2% z61EsfY9{!^+t>I)8cCR->}tGxhoJB}xhu8K(gA-J4G10|3pHJb8XF6x#zKL3iI)xn zl;%3h?gagM+xjN)!P_~y;f&9qUtfTJk=I0I&DvDTny^Yx)`a_#LzH$2fr!8CN86jGlR1*3&jrMd?ui!P{wB*}R2Z@3!u))Cojm70&i#4)%yo>bh3KmpOy zcP#3GpRa|{2;8LCb0V0Fkm^ZTcqMd!5rg5x`X@NYui^e^(p~JgM`5*OB(Ya}o|D)w z#V&q&JZla zo0OWfXE;5Miz@Oa#7NUh^_)Sn@h5|ih>(!c*$p*+#l=AE^|jG=eQ~JFrg%CH z-mVStw2)y=M|g7W$y!r&mt>rpsTvJ@4QRiYbIl8sC~00SG0H?dAtPd zOfqPPtJdlI6bs#muS5;}T11$VPhltr@*tgYG6#&KOLw!Kd3$d5QU3*!8y+sir zCxi>%sjKCn`PB44e(RiUGm-))r)wAb zWTV4Xj|Y8On%1+f9K={-ypD>Haj?FcI7suPfBqY+uP&JPhW?Q2)Kw^Pd1Vs~#LIqWAYC@B3xq@02yrZeFGgyjXpHU;s1w|<9-f#B+!nxDIeWDW z-1Kf0xG6Q=VmkIDWVJAkXIp z3hz$p`}8vPah#LLKgs=#2&W-D2jMVfdqB@1WxI>v5{!0nwb4mXHpuMDr1n=LjTXX% zAQLOuw9L;z)1mT7fNU5BJ zj1~dBh$Kqbbwa~q<`Z?TLOP*@wSXjwCWN`TID^KrLLvYXred6FP%KGPow?@~&(C8> z7`PawyZ;i>Bb-gEbfpQ>_y<(;BC1Cz96EG?MBxO5=WEl1IyEOx*8Nf5+xHU7rCRcyC{ghal+ZoUX4C#%P6I z#~Clge5dK011@!Gjw?bd`v35OF|Phs>Jv4oOeJR{*6<#!V_cRg=4NNuhOWZkuEx8` zU3Q1QCtD%b7h^iwwE^9>2kuJ6dnfXOLl?ZzzVVg7%n|cVZp8(;D;x8QM{2CbN#%6Zo3KM3 zM*~Q}TSRw)C@qn86USd1hGUJLD=rNz62STv4vA^vqS@u)*DM=r8;Vt^+rE4|O@ z+C+O~IgN!gtkWM@>Ttiu*FlAp0;Ojpr0{)>+moS7n!bVY(JA%xiDlb~QYB#3%w6mv zu!BO_9Z{9C@D{v1lqUF>4%8k(+9O{VM9h)7QT0rcayrJPuY*e!KR10F z1(l~)1)HbrjdcKt%1>fGMhawfTywFnaV4_J*NadM{_{?B6<-FPyPUxxE>G&UaeIYet8|z zJ!YIS=sob-?iiTUd=_e?JzJ#5Z|Z)Gzoa%}9IgotXPhp*xd}lm)I7u*Fi0BEqhh3a z@l>99DTFbE*(mRbwU64GA8 zCc^UAL2*%q)&j|ut38E~1~wBCDvKE^%jSs6f*~SpP}pKldS0J(L{Of3-Tipfc#!j@ zkvKr(ft_kIf~pA%94#7HUISuE^&7Z5iX#7X5KcLlDVXm>9;eK5JlPOY5>HMYgCEi@ zwq$=y;nWG`MUO|maWTt7j)Mpw#w?z!0vNCvA{X*`C6bhCjob#8)X(J=3U-zgqI||G znW*-^DEppKYfRZBMkAoO*ZUvPAbH&A#2dtJ_I(7Ch1 z1O))Zieg$}XfSj@pda0Tpm@TN-U1C#8}YXIoJw^HLVQxf zqQacge`6~SCM;`!4lxV;i|;8eRqLVQ8VTkkIb4k*9So~Qb5RJqNOQai-SfG;woJek ziTl@j!1THr?N@4e%HQY!rDhCbMkgpWMW#sC{Tr=g*L{w*iQWrd4_IAa5c-ZcG$6uX z>mAqzs{vW8c&=5de+*csYX^jzGY->NQoRsXiR!vCFnBrQI-nHXFSt5U3GR#pX+MmN7Y)MRQuH^z5J#97LCS}< zcvk#0c$9oTW_%fM>oC5RB|gENBV!!QULWJ@ImRKUb)Z-Wzm1_-I}z0>)<%&dDVC4n z5?CBG2 zg`_@oqxfF%`c2`X5!Hna*QQhKk`}(fbX;?-g*w2A)qn)0+6v?}*D0X%vMl8?IGwSQ zDJ@qdH|#V)DYIY(Gvox!I;u@*7Yt*97)8%#TI%^R;Ky5E&6M;_bDh^fjF_c`iP6m% z82q5@7fw0PO+qn&g`aI9EYQT7uyLP(UKJ?`0Yo1`&zT;whv|_-4HRWMF#w&wA{2xr z2lzEgyvQm|8=#xXg>qoB7)eU?{UA5sf+lJ!1Y+Z6NqY;NuClqdCa+LPQcG6|tx-^v zVbn)fvQqae0$oU-B)vlK5UL24+KkDw7UnB_?|?IY5n*9!MC+i0xD&4ms~}@siNg%~ zlO-mR5aaU%3QEn22-$%t=Jz57;|u)czmFP8glJ?F94pw2DFk_o0^=!2eLvLYUs6r6 zh(a|)HLU^Ha^!AMs<{VKbH6JvlNyNX+71M9RgY%6=yZJ!)Gvx96D?jWsUkJ3fsA!$ zch7fa|3so*cB3vp=Mi4MEo^^L%d&J4KO@ROAaM>2f<3_~5lW$a7oktMJ{bJ!KSnX% zojovuqEd4T-O~x0KhX(>S~ld8Xa;>K8>a8|!G9XWsxJ87G8lXuL`!~&;LOX@FHEG0 z^dRZc(oS9q@q?3N$qIvB$ss}IL{L(-6!Z$piRd5kX@xOH>T_Mx_u^&hla}XIye5j4 z=Y#7b-`TM-W1XV&T&w9kCNFs4qbamPjpt_OjwhI^I8Vwx`Am&n~8 zoTrd*!Pgv_n=oe}%n1`TZJF)`Vs=%vT9n_2Wm*d^ljH?qYi()R+;m2*`lbaJxnz{D zcAcu)4@n>L0cD?+Vp6>E8Hw3)gf(hVSOip?aluT=^GJcPQh?j3>3zBPuqLxO=%L~X zE9qPF#R1wW17@8zV;ml_F9vN6UB@+7fem_J=Rhiix#O%$nyP&bXgFxNfKuLACi6h& zmW%f-)1@=Xy;RrwEOZle1u@52TPwvY zbg7SPk?|ktuj67RkP?MxVP*~NpU}`~`YrB5;=n=|=BTh{qaPVU%Zik0I%LSN1~Y(N z0z0w9yS4}_Hl48Qcu(R=Tvi032mQGZf5GCD+cWZx7oXgdk%zlv&-R0#5?r-&j>jHk z@I2!1pX)z~>*CYx1wK`A1&Wjc_k*xfhk1 zJ4tF?K{Z#f9^+fsaWyKJ6dN(ALp=?73AgmsaeDyTo5VSg7gst;bu~k(L}x=g9q;MC z5>sEN56XL25`XJ5{LcTOqiwwkVXkV8Sztl> z@t6UvPUxt3?r?bKQ9W%%coSucoQOO$20ipWo|VEIRy>|;U1>wYh)BY;m1crm5uIXT z3Sb0UE9T9gwlWdAqgjI#th&2`3B^$mRJ%_^;1eaV$zv*F+Xn{oaAxaQWdwoKsJ^wPmY!U znGSv<=W7}V=ES(t5t?f?Ub90rogD8OxfI(Gu@gtm@4(qdZsb(G+0j<3!p4fL7O|tK ztkDTsJrf(JS}EP&%o7ldjyp_EvOMS6O`M4%m)`10fnT=AlSX!DynLyd zc_WkX^<*3ehhyT2?cz!8;wkOoshq(hll+*$f{6AxOE_E`m%WTpHYk7 z7B^XVzZZ}r`-roHVaUXOsB^pijuqu`xFMF6E<~l~fqq-sWa+`4FX=xNl^&6=FJ4g^ zH@PkhnmE*NOPYKnTRC*SA{kG_IrMT|}?`13&uiWk0NK+o(-7fH+6C zIE+Oasb1_Bhb*MZZOhk-d?DsS^a$C-{c&Qu$myM|O>P&F);gu80cnxiq{R-4BuEuf znUC47FFNej3*PVq_QPYiQ(PNHAcWHpwjvxB*RIqvD6V~v;TU3#O#@+ZO_1v4n9gc@ zaV1Panj z$fcqLHrIxRlIdcfg)XE~Wi_(jmUcDM4(fLpeRl}r{IU=tVGd&58@1f+N)3&TTtCVUiSM9PH$yWH-VZ__CGj4vm(fb}IjH3x zM0JU`UgSvXx|ZQ6BC630NCE?icMC)g%tvrwN686CW)r^SNyGL>#tccVcMqCJ$+%|@{L-N}|YXI!^u-eT) z$Dk!)A@wFq2c4>kB{AgFQZSKi%RO95ZBpgs47buVIc5Up-b2 znhgFW>@-Wo_(Ql*Ejf;xzo~Q@Ow3Vo){)r?!hG5ow6;uGQ;uU(Bte$577%_|T3AWc zR(WyOhvQACuwX%Cl=?RLPB%tkkEz@kiP-|nD>g=6%fNK8D{?o#eQGG1y3}S?DwoU~ z8N~3$@fl(EovVt)ay4xC9{d@Hja~+|%{A85xL33gmQ0w?Zh8Q^z$u==kV6zIjlLf4 zMfEgo%Q*tmw!Z46Z)#Dh>0^@xR}v2>#%v?_tt`PQIIDD;&`~&?Ihl2NJF;P2^^Cll ztGnw@Uc|RS8#U8@M;lgLj%m3V-f}&h-3XP#;rfa%VMoK*g({VW9510X*iWTi)(ONQ zKybI?a+DHF;*W|Y@nQHZ$EmboN&HbRiHiu9#2=;2K}5uoIGev1McJ65@OU)#qnkGI zNQ>Tt9%ur@f*%^0TnT43#RtHovYGo&T;wu%6iA*9l4gK-0TPuD&8d}Z-5GE#z#3MF zY#<|gORj}$;L}(OKPA_~??sD)IJzl>Ukbw4Z=Z#kggKP3|Mk!XWq{6Z6#Jtbrcb%Z z3t_VeJx9^M)h}o>XNGS6$^-20V9kdG@nm}1g{~)kQo6|R|Ey&p*1g`M@38yFf z_gEbX+sFK?D{ipgv6&C;js~i4bp0f(bt#KP6b|=9)cmXjI?l(`JK*RCusbIo5Si9z zMfCXOIuT9$qu*DaT=2qjKI|{R#Z7tgv=9E{y5)@ zKu>YsF?AA@$-}1ZEbmgrCmK6|R8u4OofJ8EV|9RBHkIS0{rMXZG~@h@=E5w8ykcwH zMuP)ZE%#tQn4@iD*$%#bBKA{}v!`;~kr#tQ#Zje7Jj)g_8w1nJVU|aOdZV7*fTvPZ zz^OdF4&C6|EEHxNrYVN~tTC=8O z=LycTYT^VJ5I0T-Y!+(~PS+`C=2kE~EEc|OtpqHrJVFL@X70fLaa4zvQd8l+&CSfO z`lcasGdQ3Z5oGAQ=WUGXky9H*Gb9W@d?BgR+=`WwY$(tC8DkJ~ z+963m^1ss?BPrPv&MJl`wJ)l&VaQdJR7Bg38f+QTn*fP4|1c@&S!oha4s1;$svE>ChW4t;8wus^D~v>7o1`z`HwkXIePYX)n3GAOV^K0stop9AwS8K)SIz`uhv=VzFHz}nC%n?VJ4Ra$ zyxqKfzzofz@89u&V}$#Y;fSC(4a3FT`28an+Zr!6l0_#x5{924;VUG(@4OkeqYo7D z`wjzsqlCXH3_nuBJ0<+D34hl$0{)+b|AU0@6NXQe@B<|L_9WnQn}lCT_+Lr*uM!OW zzr@KI;C}=U;QwdB-zxFHjqsx+{6E6*uS@ukC43p-|3}t;E#Z4h`1xV@atZH|@cD$_ zBinm21o%A%MgNQs!)p@$ZVBI)@Y`hls|o*i2|q9l-(SKHk?`Lrfd7}Q|2e`}06yk> z$M}Qbu}{0ZANE=IQ5M}dfCFX6s!uEE74+53sw*45Fvu(@P%Bj{5=dcgNpg(1B?-&p zS98|hljsgozI^mrn(=?aEy2iyQ1qD8(~A4e-moN6u`eDLh}Io zH<9}&;8rk1ZWDFFmXX3KM8Pt~HQ1iVc(M0`3NT(km-t3GW*Av9j8T8*eZHyPWHSrRn6 zG7h^HZv?Bc^cTeYybb5pwG!z#WVAS*Y->L2T8-q*rp>v>73bRA)rv=o4(Lkv_#Ad; z5dwJ(6e0i&aIF7UPvUc!N_$&ypwOKtSP)Ef+9ogs!HgzgLc_eIf_GM}l|Cpu=C?@3 z($tbexLz5dj2h_M6dFOgp=vQ8O~kv!n(jor8#1BvEFp=m3-(sCG!dYg zf{}R4{gAf6U_@01uOa!Sph-UEGm2&fqL#zeZe%p&P2{E3~-4zlC*u2Ue!s^*qH3v5;A)4;F19O4)SUk48YOxF)p*HI^q zo_g#>oja`G##fvMV27ybDeRz0>PIMjp%FGC+so* z4Cxv+{&?&?ig3sTpN=1z{0IIb?TeS=S6l4GZ^7aQ{PtfA$+^(J1mbjQp|%9#^o+uS zB@m~l74~03p;~0Y4=}SJcQXsVf|qbTMa|t*_CGWeK$`X9B~WEnfyo&5NtNT9c$@Dor_UezECTy%cf~MAB$-v&De^yg8e}p~7{^kL2+(P+C#zUkv$}~QZ z)+o{r%QQZac37kxmuY+;jeZyHZ=RNEd?4+#NP`<>zJm{>wTQF}GK~+UT|k=oO(u!o zWb>OK2NoKu#Qucf0$3&UYwDHifo9rQ&S6um2WMq%?(RG-I|1Zd_;>j$(n!Yzt(C<5 zslFtY1~H#2Vg@mPNW{ixI7f^Jk)Dbe2dza?bSEP+&HRQa3uJt)VMg%b39q zUO{6g3nDoe0YE{wMgSmK>EI;7qQU4KA#fuAwHG!p1Yk>`)}kLY(zhYG|IHbumTZ!w z8S@x((C*rSv%n4*Y$-Jg;_dL{!TL)245E#LzGFp9Ad#nfMvE92-2nA2D2t#N%TcRg zG74dztWU)Xxu6mvM>jty)oWTvJZlZ&*$ZJwJbSomCAFmS?j-2r=gi6kt6#14{6acl zilvKA*BMYUm;j*P2_jdhmF#o6_9IbPQ@OYLBWooPlT=got<%*20I3xq0A>U*>_R{o zXUD`BGOMnUR`vqyDUycHv0x4dzeZJDIO2YRR2XF^ct&2Q#bf#3VOr-j?csJOC>Gd3 zq_4+gt4g`;9A-x1hYXHA2PgZ9AHo8f2uI4WgZ>}t&ILTG>gxL$2@-8|5*szPsH2TG zv}mK!N(yZg3_62n6fM=D6eCcjt#3bsq@p4Pk|^UiG+wG`twn1sty;x9)z)wc1P}#8 zLGcn5H8H4w2m!f#zyI3jOfmu6+V_2)@BQ*TnX}J6>+G}7K6|hGTAM`5wO1bM4c?Ju zkXl1NlXF! zLaz@=O&W}7gw+fj4Xck%6Cqbze2;=6S=;hJ>4Q{==+!&-7=5A*3q@~px*Xu#9lS?lx>yTB!! z+pLHRxu~M;I6)J{wQZfF9-aMhfY4KPv&uJvuogtBReFg%)EGS_TLtvgv;w9a3y35$ z#Ay*@p%?VU9m(}Du_NEHhBg~rOuUaCffH4Zr3OPmlYFx>Sctz{VuaRU6&#jkA->!d zzo+?!Z1@MP_}=J_XksmWLWkSh$u~&n#wZ=d=hF)Mq)i^|-LUAmJPl!hMMv3LfJKMd zS%5`P=d24BJyjPCi=JR5VbQNyNm%pe)Y?KUAVrC;=heW ziSHQ)ys&7U9^zxsiT1$%zp!YoaO&PzbiW^cF)VtG!iWDq#iC38Z7kXae=eLlzS6sH zO9t-P8+SS=d^Njc*bKbspmy#J)fHz7DEGR(VY=YU9|!ny*uTV=Q@$v^G*meyb@EIBeFEv>_n)rXkIHoEOIfT|B|>jjjlA-B1^wtY!= zon9PIPDUf&7+@ssYBaj6yAEqx2~P<6qcBJ+zzz+vYY4G}>{1!5TM4Y&ZZyuJ z=8Dor@)I`3YIkSL3TzZG&D{hJ$WA^Iw=NvdDD7x27K4J`4 z5>lD`n=7KS@RHMla!xt13iy!DRR0QV-#(gzHDdl6^{(Q7oKOS{R;99p9hfsKw+bY6 zx|Lg0MpQw(j!@TIun$w`r-$s%Jyy5Y;5q`f&gxbe9NbI$08>!->>IuGhX7%3mSE6V z&(!^#H`uv91a%uMjXi0cJs{9wxf(tOsOHA;V2$FyDR{ z%(qx#O_#|_H8*JTCSGhm+s*dPL@$(jd)30vN^NmZkS-}Xu3#ZFd2 zx{SQ)0|Q>79b#_GDCUPo;||L_oR%bj!&5&`0h=b=fa4nF=c>raDH;&55S0MP5kL&a z*z($4tj2*w;o2XEPz{_%8x7wmR!fZafP|1oy! z+G5RLedJ~+4{U6LlCWE3-`LHi8Yzo|4284)1$G0YT_#d$1{{SY<0s&-Tr{)-72Xu* z+{%Z!OqMKGm;9O#qI(yG>kivbi^}68<1884Bz;+G;8&$a)G~_^_ZfQg>{M^GE-s6x z7g`nzts9c+{arb4JY_H+d!f&Crmr5C%>9BEW9cj9;hPqK_BdeQekFX^ZP`#slu^XA?; z=ATY=-XaWx3fu8*{C43BI4Ab59P{j~g{K`?C{M+L$`Q}9!B4+R?^#?i`&Je_yRaPRgr{fe zFyd)0fwP|G(xm)?r`ath?T{WV-5#!A4kC?Vt|ohFjxcDr?q}ev_8vrBN&4Ql#vE1S z_3NuI^AfMif*pQ{NOzH@qoScUlnisFWFQ)VSRd>XP{WnX4=V3;oDTKHMk-Ho@s?1h zfyssQ22~D#VvzK34`l5VeB*20LxNAw1Ub@$r@cwmMWMvdConoVD0&hjTb!J6!iBw#nm;G27%}E3mzVo-^Ct_%!@D2S3%pe^2ngh#35P zF95%CnZXZA!1C-`y)|417CVFy3Y!9OMV{to_l!RI^pX=(T=4*o_5f0y6~IQXs4 z1OFD8i?I!#n}#3i;4gOYR|_^A$l^ZUX3 zZfvsm4LSI43H}GJ{YwOYj)R|{hHrH6D>*PeM+&~o!GA~a$2j<#((nxq{%!|<(Hp>* zJNW$tziElR@2E6~mP+f&W;g%S@MRAE90&i2`V(^SPYC{Q z2Y*N!KF`4)_=_F<5rTiuwZBO4 zbAWHiu-}__n|QGzc~p7ov>mk_2S_r_F-a?2D_3PN`HMVnz)#U!E?4rNH3)QhqyFDG;EnCgBpCU?N&?Izp3+`}(>(`Mt&FP-$X^fuM(wz2S@CKYiPiyS# z7xeU}^z>3Y8%a-x)6?Oc`JJ5Xit6M9S5zm*x}rMyV~T-(#8P&`VWA&EArh{CE01yl=ZS8`oRYJ3-eXHbeJpLU-ZDXSu}+^H{yA~@H! z934%x>nn|m_E~_fs7fq-1EU^Im0Nkb%INKRDDG3`rh=Gab!a=J8cGeK4mW3WSd#-M zJ;8?Ec0`Y0Fr1ggZ~tLl7O#fuhnO14`d>~*Wmc2chuF*OF0xCP6O*<%eBCsi47fZR zzHT<9z@=hsxONGr6{San>o*F5samYAl++GDzjF}ZN6G@*=;HhwH17;FJaq-Wd2Dcz zrwuN4m~(~gaA6uGxmWxexr_v{GW#>`XW6);cXBQkWr2$Yf<11fsh*S9^d$XOE*hL0 ziZmLrq_>f0;h7svUQrlJT_f*`?PLr7hd26FH{9a}*MF!n8XBH)WiJg)tv*T9j=Zd< zjy`V82PZzNNVM`z;^Z3QJ80}?_Ly)7y(yES=1(AUId*J0r7ZYfJzO2+SsCss^=cn!W6^XUdAd-5X z0Wp7N6WODqTk&t;<(BH6KFu261GdW}LS7?d*^W)wx!QX+$kSBDj^gM{Z}bOvq1k7M zI_sV+Ex3A0DCzHfBBCzWM=a@lB^k*o9-?n2DJFzZC?4GYOF3KXNsZ-%U{5&Q(w^$JFK>a-`-?0%U3l%b5!F?NA=G9 z-nHx?itFy;8e;GX#C3`@zEjtSgzFOUaq015^U%udI#cY_;awWTM7E*rQXm5V3c?4- zrpMqPC+;<|(L12k)Nfc-Nof|_wyV9D{Bz5gf@W5gkVIUbjeZpdSQGI68HYrgpgYAaF6+bQ8_xWN1x%7TZR6OFJIuy{d=doqBdE_v<*3$5MUa zw=YvtQ!>khR>o7gb7gzYBOFNjDEuiZbN?)c*?Um2B=OSLnszVrF?pFaDX~5abMVtY zR{I%hu;)wwX-5wf54|rVUMlxlaro+z+)qL)qN(hsq^f7<6Y?#iZ1NhU>GE#mm$GzR z-i`cHmUhd#QJOBVL3%CkM(Mb`25Gds8>QXy8l=neZj@fjYmgSpyHOgg+y;udH{wp2 z6C;zt?QQ+&mlk9*67gY5dSS>C`8JiEt0X;IvvDPJQ})s>>vQ4V)s@50FL4-weu={v z^h+H2O23ppY^EuZM$9xN(wLd1MCv=!DUsII;BEb;Oliax)Jq-K_8{mg*+c#q#VEAR zALEn*g;h?^6E+ojh6I)zVB{GRD0Tm_c6J^lP%8fdI}3>Q{+#(F@SCnE5*Tqsk-$S; zQ6%s{iUA3%l#75!pt)m|sSD1K2o)rIS#XZtUBxslG(!Qjuzw~kbl4bL7*eymZy}1Y zPf5rfO(umc!%yau9B^vs{=%EtH~K{KDkFlCX!0tsiXwoMSAl1YEaI2zCC}*Zds&=O zra}odC{Lsq@f|FDA~{nd2T=vJmM2=vEiCTcogyt!sGNCG*aL>9JuKvmD@X5~_Ae_u zi1p3mI38|hoyz9H&bqUP-xW8aDjtvV* zRq}4Qz8dDovhv^0H20)w`iw*;qt$8L=Q>#T=s2si?xV`^m-gSofnq4@wP0>W=NDXg zXu%KhZJw_br&^~YT{SCMViQx;% z%g?CPatKwd$DcJ*p;r}&5B4}xC5kN4{sArq6Q$gS8nZ=6K#dD9=o84L?#+!^7gHns*i8sRa%1;U{-r5#1t}+8XT#hT3d^H?; z7;R?(diar@1?b@n&V2N6nJWrCTvK_q3z;HVxf$z11DCC))5FmudS-hYv)Pi8Kt1}V8!~h|r*UP_! z4@S`>nxn#(dqO#-i975uljOh70WQDSb4bpo9BZ1j8=gUOz5Eue<}_Cgo9t~;QzD?_}@B7zOUd9ckt8G@Yg!{+Z_C7>d&1He&^G` zzyGB5r!ozHo`b*C!Iuj@=KAxT;GcBx#cBBC9sH>dzC!SxgTGVo*E#r()xrC2eZ=0k zmxDiA@Nox!nc&ZN@XcxXMGk%q2gYZI-uEmAKTz;rcknaQ@V7enKRNi91V7Zl=L&w? z6ZXDi)9{x#_$wUz0>Pi-;9vV2@QWRM|1|t}9Q+v$zNg^Nb?^@e{x%1nlZM~l!5{A6 zw=Dra{!$0O_Qha)UY=v)^F9aO_e{0FpM&=Vf2xC@orb^9!9VHXZ&Uj(xbNFr z@Vy-T#5DXb9sG3;zI`e1D;)fWxxlY^+{P!GhCkiGpYPzq(j?yI`tyw7|K#8c)9{Bm z_^&(oH>FvtbMQ9^{t5@*x+-|z_a1ihp9Al^wHf$7yZ($2{231ZsWki(4t}wNUnls# zIQYJTKit7jPs3m9;BRyAJv2TAu0K0jDy{Fc8wKmJG7W#8gTK_lf28&gbMVgz{z(U4 zoQ6N%!Jq2jUlIJ_4*pKTU+3UERtE38^&vO^9sC5rzwP>enc&ZN@XcxXMGk%q2gYZl z;MY6&fr9_KgP)m(ztzG2$-&12|E`1275uhGcLnlfhW>6g97pZg5cxs{tyAVCtI?mWowL}{;ozVUOeI8KJ zM5}k`9`f}jc1GD~4??pDZRgJ)Y33N>xLi3R*}kjM(#1F;jpjDruVXVc;rer_ z&^CSWOxv;g5n5vp{CN~gMf4f_>6Pe7wUebxJ5ELI066cx{h~y{)E8LAdAb{$AwIA@ zsNl|(vC{LR;rdU-fs8&4w-xyjjXteXg)#~wROnBkDw@bHs=azlR`ven$?T%`T>VOf zIxv)#^=AaCJvG{mP=lGNbBvAlin0eI7qEx>`FZi=MR>QK9f>7>RUAv6QKIH^1}Bdn zlB|B0c(oW_FfbbE1V13Ieb5SeNZjTKHbks#4|(B1YmHDNfS}ilCz@lmt*ULLmhQ4M zPOP>~6^f|!M2zQoIDN#GCgU{RQhykwUTvPHXw%c!zN?3hks;jP#HTzPS;u9s#Te(M z4r;nTJB#ghiNHa;uNF*DGH)vX92YwYRGq^5wFWIn43DQbChVL{N1gWPVRNzg?zr z#cPat7hf#r3)1Mz11t|7a({~PN!4LiHB_ZQY1*ubfSZOit7u_$A0?cK zIV)6n&lDctet3iL3o%`tLEa``(mBjn`x9&1}s?Lv8A_U3p3c><{pM9J%YEx(l&*KrB|o$I)S|IT$>!hh$wg?Hn>!(AM77YE(Nq4%{hRDeqP zbYC1zCG%u#R?Wd}zht7g@sc5&SIryThaDhyN4JshNrJY+TV}r61Lx`})|{k9;krFdC#W zLp}DPef&4PKD%R~9VFSq;p}gE+C;*b@RM5WCQhi5!mOgfMI+tB302aYRW!I@q?^n|>^7;-DjHlY(oLKIB`sP-g9}Bv$(+P) zlQONM!9^n7WL{#oNvBrP-~y3uGB>f?sL5`ycfLt#wrJ_g;rfvv}N+>bc$+)Ht4n!B9l+J^u& zHx2cqgKBnAYXtSYpl~x3cv(+<2v#bGGuKYAM zwY%`>b*fe37Qp6E=^4tf=HqK+n8A621OsQ>}j6-`m694#*a+Tm;I>aaH+_pRXn5fwKJ|Bvu)7XBYG>}KKr5xsBbQhr4Pm%#rGTmt_$a0&e1fDaG& z|7GF-PlW%+3IA)qn*jf_MKJikqR$)Jn`+yON{J@?k5g`rS{wEhBwn3w$!zaz5$V#rYhAqqjjAByyGzTE-F zDjGF=$qhyj+PyyOw5vJ;^c;qnJC%@eibhp2yRGmlGtFWm*Q^{10>e@34i(BQTD2Q} zkpJ+AAY4^$#Lj`3Dn}pWM@$tYfhtEI8GjT`X@KTehWLyMpfpZN6QcC45}3}2Q(&BqMAC_BvHQrtNv*2fG& z32tGV0+r!NXc3R4FhhkZG443uSLEX|%j_bvqx@x~wYz+OS-FqHM)}UN8yQ&37}{h> z$b1J`V}g9+d5eY~)B^%3H@8HQ#p&SUmm|H`*tq0otc*yyw?KOsFRS#SvL^lfJTlTiH2C~o$Im!VOk#QZ%%jo@p-`6r0>URY-mJJoPK@*8?F`JKrnJQ2rzA)O<9P!_IVtm#3k5!l=ud8lhLkxOuhZ)TAam1}ktc}tnDu3hVqJajk zVC>H9yu#jL=DmvyGMb!NIcZ2 zWb053C`PM-Ii~P^6*!W|wgU48jp#3OFz^u!j$|!}m2Rq8PYgx3Za6>L4v?Wt7kTkM zTVkPYI8Ece@}#_Cg@=^g9n4SK!-7W?J0esROHGzOVHbvZMYL#w;~)@Eou};qUV?%t z7j9^2JT(~|#Y4N|p^uey3l{=*m109Z76nT~J3PriJLwLv{h$#8_s4`j=`NxCNq2zk zsp(}k!uXT!0M}EqpVbKA54@vkzJnNe7rVRTNBd?x^lDtcZ^TMptGTc37~%V5&r$yf znD44PjrqQu1I%|HX$j94i!SDSU>fSX4(dz?H3Sv^+@k~~=KHWTRDpv!&Owz5>S&-~ zzOTO*=DTK=G;qkv8Tz+khST`()kN%(Gns&XoY0is=4L9Q$zK%4lEVuXIDCM4`B$Sd zXmzntgzOcx{TEOQAok<@Mslj?zUuO>L^Mun&?{;o+gYb9GOR2A}{n7`ErDY$v}!g z^fFNiQ35FXYzzFTrY$jWIdZUlf&jLX=JW4VA^~lm!XZ3uHoxHL1KqwAVli@y;A8R? zZ822a^QCgLZ37&hh1N^ox6;WYCjXcw%k*E8e@v4l{ny=-hD-l-clB@&{ny=>4Ig$L zqS2V)zW!?}m!ODKxrF>Pl}pG!Q>Fj%r2i6Y-QOu4YrfQW3v!0^i)>SrSkwo>`eVqP z$c1E|pQ?WvhEy)$h(?AafP3IvZC=&moku75zE1r>8F=M?RSaH1$`4!W3pt99f>OiFqRBmvUI#8(tl^b29 z4pizurNTyNQwJ(7h@@hM;zp>{fl7-ashA=2Ih8t4sY4#g|ESBpURBxC$;+POmA(+J zKTGWva3&jc!xI_}&rf5{gMEH5jpnp3_MJYO zdnL~9hk=rw%|0nRNvtcTbLp|VVmd#enxv5^S!1L_5+b5uhw&_8umzc`(!T6iiCa}s zA_X`oyb&(QRu^8L&93K?V_|)6<6T5;tBDI!{8Y&eak*bxUP@Hs#E>>tnHJU;!4E>i`IUO+hqxd zW&4i7_v<2mk=hQW8&ioLpsRlwC)JL*NfJy>0^kBZ;wmrnYScsuBg zYA%jNDwX}P$yvHutRJ#xuMgy{smNQikWGfWU6$@)!xA!ad!=vHtoL=Dg=pT0MjRz6 z3!Q48%^o?HjLh^#pt{pAp_^lnq&$0IE)Tq+$YI3uwDlS(0!B)r(=F?;?v0-5T8D}50PiI6M1gC4UuQs&E{nB$-@EsrWppmpM(D!yKm3^ zn}fep@aH=C1!?$s4t^B}+Fz;mCp!331%IrAzc~$mql3T4!9OARD;<0x90}<0)6Awe`fz!rO1L#`-h{8&GtH-_7Bgt zo_!!NdhKoha58B*DW4S??+44Xnm^C}5mhG1&|0&9kkxxZI*G+|JXy#9Rut<!VWkLBkX`f@0sXb z$fx;w&0k`P55@kErAK_+nuFSo<71ls1Ge3Sor*r>4Q(eTBp&+AWPln&vvi97LFSFg z$EemDtMrRRfRnhyu(|Q)!nuW*KmIE7M-)=q#r^>j%YG8{H8T5A5E@BD*hL1)U-KS$ zHnp*glkYX<(4CSyML{FiArHU;#rw2Nqs9oy2=ahXuFYc^LF7J;AmYzrKPg@zIxr*y z)ca|DThk7#5tuwi?rr~|s=NJzs_ym=s=C`hsOoP2psKt5gQ{-!kC9#NAF`CVfnWCFvh-+IwDNe~? z`9c<#Ydc(!3|K;@4l}62?m0ULvlRDYo9wn5;6W$;%EH{27>E-zMt={NnIBu|p zGpM?J{b~CJsn3nqp3NA<5+4BObOoLzDD*S)XWB1Z>Rv-bPjMHw`}T`xil%VRP>Ll1 zxuUScYA2o=#~w*gygr)Qej!+CTxoP#BJ6|x2~X|peAX8@Ml61+!16$iLoFK%eIO(< z5n0>J7FaXTM0^5woudY1#fYJLpeVCNACsA%ie}6N^bFlzq4{U1mw{cv>>A-&Aq`ZE2EnG0Tup`v;+{kTBJM;TV zHJsjBmpmdh5BWNK-c)1wcQ8TwH@onQfxbSo@16LDP&#rv32o%W>(^EP7y9{LWIy2( z%(1uN{1B-$Koz~I!G9;7=t%49TYY_fGp3n9A9Gb+(aj6zT`)2t9<1nAZCMV_Nf6n3 zgr3Z9kogHlQvNQIdvZ|MlM4w2JWoem&qfNcvqDe8Q?QnV8%R;lIwwNrg{T~aIu-Tw zX$|TQ=7MHKYGeqoH%ZVU`*dg-SFrCUOp!rdN7Jh# zi`f47s{Apg#-IxP1lWGA&h!>V*9)qbz3L3q7O;-GFuFF$uB+@PuLFG@rG*g^fmK}{3X zD#tUvh-JL>AJbj>`p)m&MSowrq0pGeVkB5!!F@=yc41*R&QUyK+ApTQaq?>f1?LHk z&ZOu8N8W zvODjvZDuPT$my)faWJ`^HMy=Pud^o4)f9Br6u6qg&YD73)4Q{#ceJ*N0L<2K{TOM& z-h>M=^>^p@R5Dxs(*6^0KNi@VXMw$W!)lEms%_diys>G^@Fh(fhOcT`KYX3|9XSAU z@UC@C5NcB~LD#w%-~~00li{*m@zdx-a%|>Z)FVCf@R?m0?VNe|&N9KG$#TuTa`|MH7c=*; zo>+A(kGZ!zvBdHEh0tE8A|xfToUNv7-!irzZRZ|EBU#G(^!PBpJoa!Ytp2rTNhF#a zimTG>X6B8}nrJf0be?JFFo1cz{1j0}hCuui!XVT(tr(x)`ZJOo{yh)~E#oz(De-XUeHm{!?Wl}v880{OYZ=!v-m8%J3YyIB@`>w@ z)QTbf0h~Gl;Z*3HCRwp23B2dS1g7e|Bsgc^7)_hvuIst=Gf;T?qd%@aCB4#5!*%^= zOtdTeQH|~RwZZc%0+5=w+VbpyJx`hyEZA>Nwgp?pfd$*~7Z&VDw_yLsT5WBZwrkHm z=)mX$EdL6}bUT_FYpSPa2@+GN+xBEVt@Ce$Z6 z^C^thc^3-9o?MncZQA`kA0)zn_$by2?Q%9n|T9suq;&_vfXdhB>I8IH-YwssRdIytIx_ z`}mZ-?f2m0mH+weyCX4_BHZ_QCVJYwpzZj61H4xCzH9d zhvXidR1k)5rsB(Vh0lcg=Pd9mR_^5KAb&7))On`GQk7&@Alu@5y$CMt@KTd|>p|?2 z^_@(5rq4k_M6&BjVqhT^%ljRsn;d;ap2Hi4>pnFVyx^p17sf88p)Z@f&aFzChjz4d zY4sy9((xyDoz$qknCqAvAl%O?=V{Zl(DFE&pq6%2Ki{gcX+P_>F9ez%b;>kQ(P%4P2Amf$9L+L3v>v-94p zt}ScF;y9ZB`m33gab>_CMB(hzh~H%?puKZ`PrWLy;15BYC~xDX@6~+Jb|}DZ1%~Th z)mrQFWy|=o_OZevzb`i;XxLsiCO_Htg;I*P*`n9?rEda~oZEi5^leNHrCSL# zD`_kfqNMtM>@j2p;Ul&mf|va~;?7CK$S01Ke#ZBOf-+?KzT+=%Ab+=a>1Jgy@lx83 zz`GP#pz~yF*Ap}romX~rJwY(&yt2FVB==Ik9IIW(=WD5Huumt3_N>&HpWw!*{SQ6M z_Wc`B5z%1JlRxDQt{cuLAwi=hBxvM|f&KigZWg|C<)L7Jf~LU179T#Nu0a1{#qV5iJ$P804yoIfG(cpfj-l?czXlvL@fi zWSo3MMRFuKfGpX5_&V5x@O3A7kO@nz70Zc)c7+&b+;5Y2B=9bu(NGGYAu)5w86pnx z)M(Jq>*a8bC~`X3lNH8oN&62v*OTlf{bSmM)9x}OADbpCH-82)S8k>~woJ-@^L z(%Q87f+fOtV27o4)U^#(tLqyLYv}rCmGkYsE$KT1*^zO^_8+viBd_2uU7K|BN_)-p zc>Ra$nZrCL*8Z<>H37aA((@kA3YvnQiI;v_)7GYZ4N+&A$3Pz{3*WM+Eb=^FH?i7x zT1?f}m{?HzPA~S|jct45tkl3JKA*V|zxs2hQ`^kZbZW=^%(tsQ_0w!2M^&CsKxI?& z`CE1Jz1ujE=y}W~4PZ$7dS0?Go)f#g0b}$z?Mr?|dWpol$R0Qa;^x!0h|hMEk*Ci2 zK{9J)4}c}omzz^o2WDPt_-Z>hVkRalXgu*suy4HlRbOOd^mQE9bKJo3Q;xevz0D5t zXs}Oj9<}+JWB$-DH0tfgyH$*Tu()i6K|FFzpFi}=A2pstt(g_%Z_TXuP4|ja_lo+= z3i`8{{=9%Lqs94ODB&!Gr|uBGQ2Tx-Er}(T#I~)Bu_GD!qT|>*n~A;>orls9m1UJL zh8VMU_uzO!_K}xsK8p5Pi9Xobv9IS8-6MN@{+5|7x->fY3JE1{~v#ngdI6F^u z_-X!UMM6pEb>J`Q%d02c(YDOCH1_$^% z2Y90Zw*qLK;WueDUF_}Cq~FW!H~J!!=5cn=V}<2_0SC(ut_>u~#JMW~J5 z=@swuS~NAz5c_u;I`O$U9Qa6_Hc;>iKih`p!snoJu^3}oG>;mlL^9tef z`Jac+wTA<(h2{&N7kA?G9EZZd1SM2dalFi#4>`@w^u(XhV&$G z`mSE%uWS$|pDlboV)kC#r(HAV)2?Umo^~zGdfHi<^RzQ#JSp{_-IeWh_=gcgpvX81u*aVqpKdTIEbuj%mna^ZKQ)O^{YY5d+6 z;P=LT;P*!1cXoMV()#$_&;ZUf!tXh0{Jt6-k3I5))$cYPxB5elTRH9?_11&Ms~3-Y z>yO2&Ul{c^7<^?K#~0*}dK(PxD#nBKvq9-#@VC-9zTg;3!|f_g2I*&00S3>60E1^j zkgYot;?tnQwXv}Lgw-n-mLIp8?Er=4qc+9C@^6Oga)sq{JJ)+BzTYl<|F!P;9(g){ zzfNpVN-rPVFEDJsy%XCzeg_scBy0~Kke?mW-Pw6y`Y%#&`2C3hzn}Z-02>eeFT=(c za{wD3aR-=e2>2aLwx|Y7Hs|uRK62O(jkCQCG1cd1bH&S!EWhZ-#>t}95Q1ta?`R1s zj=VG$LIj>-xQC!oC*=%IG{#d=xXAfvx_x1GWN$1#JMwzIQ-vGf@fq?X{0sDQ)1XA( zlcjmUc#M^+bV2nLc%+j)B-LWGBbL&}i9Sk2H|$YUi;Z{`4(_6*=zEq9PHd|P58hP~ z9ZMH4Wkq3!czD zf*+jNLdV}N51+XK%o5x4Ld~0Pe*=>E6Y+}Ej?lNL-Hv<_R*A%AwePiHbL-i3vEhw_ zUgs#5@6cR6g$hUh;i7Ek-1(MHPuR(6(~^%JU-&D4qKR#Mur$4Es#!h1>6+C`Iljno z9mjV$emH9Ur^RJkf|YuAzmrFe-_^YWj&5Nm#D1g3!_j3{fDbY&zz3NX;DgKxIJ&MC zO`ooQp=sCZ#bmPg`86dCsXI}sKNpGCn^@(w$=hj;nx^A6mEO4 zoGrl`TJw?D=Vj+*{DBNteKvyTHbq05*)w#xAiMFuKD(D1c;Mn}BC4~p7H600K={lT zE^Ws2&{6YX+vUz;c4abH*jK>FlX_)32Nrf2@qcqCx`q8AYr1vqe@VGjr^jZ-r%dPF zY@r`SE*Pi6>_bVh2Slnw;W0$S{hb>RG4Grh`!w1nRB~LAYfiaUgzKN-9D4+WJUhIgU6n0z}sq)xuYm&$KI**Ez* z;=CqLBg-#3KvpyIy3*7Af}(}G5J=Cv!_X1`p8P50i;>uG7lY;{h#PPb?8wD=3>e&B z6%M}gu#npTNpw_{+co)tEryjM+qR&dMTFSmLI{bgk#R9XPF7h2RYv8A!qTDSqX#Dz z#;UzBd&0w?{cQV)=KMaIQdo z33rN?F+UvOimmB(oz8Xfg_TlqmDxikvUj9d%}Ou%K+zPg$3plED=d>dkMI>$ z;kqAy)YOfMyw^sVzQ;y!gqsDdvzT+KL|q%F-9D%I)kAehv2z)X5YVppEbTQso9l$h zY2Om{(hCuO5cT8n)rE(t#P$$V^g0H8FY$>Si)IS66!A62r*m5|qeJ&QL1!AdPHIK< zOAi2%aGCEWkG&Quh=5h+)VMsO60a*x1All?yu`ZpZ#eodT`S`r!Gl#@(L9J_1|e`q zUp!nlkcZPWOqcT*@fgxpuIMIu3g4}V=r)DYGKhGP;65W=-yn@Qx;|}-q^}eWO@h1U zAuH_uI)A^ERz0^v4Jd6i{#V`r7G$_?lwLdhP`h>&)f2VVbq*hwy3dsGw#Dk6DJr8* zeNYqX-A0Zm2EOaR9}kzGD|!KYd&M0qG$4lmH_vsR-k(MXJY&;3T<2+>Fs+))zZ;2W9VI}BS_bjVnJG0%=BjMM? zHg>VZi{WY7pUZu}uJ9^X{5o4VS4_`#XF2XH$DQT6vs`zU=g#DO%zXv!tiYWWx-ka=6AM{*JqD}md<}a_8!7mb&lvj~bq*9)jQlwJeu#_T|@gKmJf-bQexL?lsSBt<;V7d9eKZLZ4WKr|N$NL^x>-=tIm}K&-RGd5 zbWqC#bqi2%>b-vkLGN|RzVp4Ub!7ZQ`{=y^X%gy$juqZu9oJtwShiRBCrC^Me& zivOIaW$GxiYQptGRNO{Rv#TKJ+-B+A2~kyKKlo3#9nyBaf7`ZltEst08jN^yP{eMT zXHSBNlYh!ra;&P(7ZE2RSWrup)Yy2yA4^-UkcUkjIize}@_!&@fy=8YQ zZV|H8tmB@erv)G6VsWdfL&tMqsQ-F3gE)^tY`YOYEauFr11YK-C8nFV9AE6AOr>H{ zs)RYm(zpIm7Iuppjxv=ST{)^-hsOMEDRbh<@nx~(&!XkY1L3(3QKlNh%m7s?u5=c6 zi5Lhu3wh48W3^}&`qXONg~Ii}R$VIl?;O^UVSRHfHz0MKv)mv5_*-Fikbz6DNVT-R*=ZXT^E4Ra#_@)8v@DI4sq? zxA!|~YwOI>yDaT{Wm_T2O2pdmj4%ZYE!*D;*fV!&uHgX!FSD7V9*#=L()5dOh|X)- zPs3AIk?4T8Rxuh+>?BKWhxOA-^=;BiilZQ;T?C_n)Aj+~_-ARE`kN8UAFxoH0{&(J z>-(SgkCwg~uKR`h03M!V7qB^i`oE2SBcSU>SKZiqA%%6&$x5#jV(Y$=D1h#*#!GEa zjz02q&L%|s-5wQ`RMxhPs=fLLiR}v1{d(d}g|KXHJ#Ex31h{tz>mNYgXIJ+v)e*)N zoFCooJlySk&&8SWA9p){GkwlZks@FkQunBZT#~X1&%0RZ2Ins&E;R~)S!>MIC-KB? zX3WP*{LAogN%)pPPK3$CtF5&(;DtVwjM?6HpP@+BO+5qXBBRAvHS#zuqM_0vjuZQd zI50*LHkZM~%_x)HqSlL$^5An3IT3M4AWVpG&dPif^l^A@WiMDK)0^*&JCR zZ=%Q{DhXEWWk>{!^D_MOVu|NO&owh*eOAX)6RR{mClcaVrTUH3pG15Cq}PuVqf!il z_~oZKw)by{5_O$N(WbNy#S|>0ij)cZW;s@TJvzX3KU{Z!-kFx{Hj9w=d&yIZ)fDC} zE=MS88R>r4Eh?YV+xUUr0(zKpAtpVK4c4(Y^SN@`f0X721YDvPWH;tZYO(d#Ihq(c zXE-b5xY#I1wXaT#)xkWZPS2t$TqhSJhVg@FVvU-#QKfU!tto2BfUY7FW2o3Wd!!~G zuf7SG*DM3QjEj$ z(bb_88DC>D-EDktXyq-&pUJtFem3Yf!YuQx0d(TxrSKxt+1ne zH2m+mo&4`C-6{SzSTT1N%~loC4pq(E=Y<|~TscRztA0NWhy){4=9O-I>FhQ=aXME# zG!U$S5=kUebPfT7Y~!S^mVH3Iic}T!u31H3^)_NiS8&u@=_wjnb%>EYQ8TH^2GdPp z$z@}48`hE@CzlY0u@TsC-LW=|K+~Xshi$q20V2g~XnA*F+=+^ccLhDyXX z@g`~)au{^s*N2h*R39@yGtU%F(THfVFm>ah?Vj5wH%fwXkKVL$HC$IgZ&mDKn+aI|{#S**ZcGf#A`B7>%l~lG;Hjv1I@?;{mlpV? z26tTAh!o%G36Jqgy?HFfrf}zjvPT zh-YztN8DeEo~+|R)J)HwN2H++cTmL+YSlQP8sQ|3wg2Rbpsh_mvbOefP@!J{wL(y0 z?U$#amN}@`IPk=c=;G(D1Pa!^d?c)W-wVtmJw5;TVNX07tvm@dP|=kn4~eFZ9R5R| zJ0SZ-@+QNakx7@VGUt`W;agdWjh<2>%ORJ1&Jm{Pi!Lp)ZRP45Qu=WbRTQN9tTL4O zx@&%%Ot|%5eo%4SB^LLO!1;5R5{^0WIkW`_LYSPeRu|Ai+4)I$EWCX*p>@dmq31wB}dA? za&4@~U52=mmun0*dOha2)5)L{lu*=lcy2E%xubQiUj*H}>_6eKbYx>z)AlBQSX1@d z`wQM#!Z6NiJMxDWVQx5oUhBvo%PlkS(6i_k%Wi(t2O}Hth9?bm(+49O7ry<|+qtW0 zGrsH{#dK0qboPDbd+H8Fk9<2r{DQ{rvZO!bpW{KTm$Us0H+CXy#Rd!{_-^7G6(n?2U`4<0N(}BR@|S` zP=9n#PdKPs@kg2ao}eU5PfbJp%0V?as7kfuu@=Bk>T5aZac%)C6S$GmhiD0hJ8en}=&oV+`XPM>M44-2R7)U40~q z95EA&A6$xr>mP8JAQp9wEAsDRHs7dAkfAbvN-OKQx{xXC>hWd%PJ;LA4shITo{=?q zg^GGy*5oEjuSBo%Bq>HFKkn|rn;#GUG6_nBG#pKqMb(cyDn7VMXHg+IK;=pM8V4+)rbp7c^}rOzX3ei{y00to&Zy;Owk=ga*gnG<}xutLnJ9 zlF1C!4$t^6sn7dt#B45QvDS*7Dc&hOL#e1@sjFdnCl!tL-Opl)Z75gCq})-FSVv4@ zJn!eAhHF~{y1I5I|9J56!DS} zpuPZ}z2>|t9cGn5HAIl31HGSBslnhyHVKB5c$cKoqo z@bu(l;>*YD-eczuRHBLRMa zKcw1B`!imj>CZTi_^ap5pRpPv3NG!@6#A;rJh{8UFVmCU8-s?BBVwg5)wBo5!qB4U zCo9mq_LvYhZ=NM>zqNNPWBBEI#7n9`cw;{wAb5X!Q~Y*DtPq);L5Tpa!0TM2fESN{ zjZ(L3`1JVKC}mtLxZb~1Fr(|0C=ra>d^{mvkJI(Kq%$hG-j^@*>N9Y(Mn#>bC-kpi zde9b)c0NTx%%#g}-e~$HjX4)DoHy~r#_?ZLuuA)nZ2UAgpJFUeSZBZw0u%&3n!!oaXL&x}wRM{Gu`GSAl;k-}jXp9Grh!chg&>iVI*7vFGJM$d zSxr0h^{X;Imy{J8H*7kwa+F&r^G-YS3#Kz3Bm`(1zoeq{^>At>gxIi1zs;mO2fL z!2pnd$)`3J+V^e;&T{3o>)G1tot^ByD*K{QLyf;}i<^ClFpaHLsX0V##pk82=&jFE z2w%8D$aIR}Q|IT#5=w~K5lfws%cW+mYc1ti%6qa(N;a2=j~Vm5=f4+xOup>JoEY_R z_ELx1%zowQhm;1wa@Z?H4h(521|-m=r0GXMy;z+h7D9OzEb^F93i?-(1>iXoPiyJmkEWx)+zZOYF@=M>u@# zM|(}i_mk2Pd$9FcEQ(RW#~dA@gOCAx0@I;IPvRwQTHs2^8);_P8@aV149s4ON~cAm z(hbbyo^YLUxjr&yZ(r%dbf@-f;0>%jZU712;J*%-Cm?3pQ8mGIG@{^kRB8WVm-(k2 zO;^t{NrJMhXw>0-)Ip;HT}VO|gCEd>aH_+|fZt9hB`ARs(9VQhp9~~;OhfelJ^qiD zF8A?2Y5sm~sK0YP?i2ruX@J017xn)I{c!l6DTxSQAPY|qGmURs5!{~h#TSxq8d zzuwH*)#dbU2J}D(Acygo{zd44NXx&V2Qa~!&4is@(VB&arae`Q29>>=jv@vpmIn9( zWHObLOkzOldLfNQ%~R0>(E;p5W)FyrzraX>!`co13BF$D@U`JZ@U_q-tL|-(Fp9Ts z`k;N{>n8!NNW6;1J(^f7{o&YXpEc1`_NiKhT3smO!K7nwPf3yeAEYaPWFMWzQhy;SrPfz+q*lovSr;SU_|3ZC7-zO^ufprT# zvZgzHc=2ZE_FSd}9r#!{Rd`_Lvltnp-;RHA?YGdF^rtX@=G-x*v+*h#S4wt=pcMaP zF)@}=PmK#y^asldHaC1G1OCVKnVsAv9+UX3&`+!@*VKw6VTO5FUqY+}fy%{;CY;Le zV$Gk%INAheOV+8KvxRXHdS;07v_>^#QoX8$G`A2-WglfJBE;YQH~57zy(Z({sFzFg z3zf{g6aQQM!V7NxX`&bh&E$Itl#27iCCXngJoj4T53Wt~2TyX<*n(V1rEZ$6Aw?_R z@z{KJ$NRj&D}Tp$g!Ig_X_h(ja5u@AbhE$Bq)X4YvF#@cDM$koOg1UeqyF?7YM(72 zxBG2${!YxSF%nRKV@{o{{mXACnxYr$oKj|8vsUhPi=%TYRj0Hn1n(u>MKx6=vrC@# zBq@VVUFJj=<3w!oL<#eaTyv{Gd7jFgJm1#j!Cy)9XUdsw{xtUMI)BD?pFc66K`@

-=x@J%wca@0kZ5VgjSD0cg~vP7Oc?cEU`{R~Ocp4OF=9`45FPTyy%&#Q`=_ zf7OteSgvtmCJDK)&Ov?cKQn}!=AZTPO3XOr3+=o#_{#jLD%lQJs{?Mu__G$;RF7<- z38h`8jS{eaZnT&@HH06RLus-Hs#9atDZ7C!#f-~BlTAH{S&}iFBM36DE4`np2^g#G zfcrt*nsBhs{UoWCeLpncRRr?I{|ou|1@S*yk$+PJ$vOro_`ZAdKQDKafBh|mc|#Iq z;o3T(H@RDk_gU5U8(y;8_kmGN?7(~2K{U|*uv^peFS~7ZkudFATK;9*t?t*-@-O>u zWnL?|HIP|v>v{!CH}da`#_v>rCzh)I4X?W?ni{4f`L!=me+n9cdvDCRSHl154E|;T zKI|Z*V~~dZJb#T|2U-1hq3uP9V}WX*G7?jaU!?wQG}yuBI04E)HLN`3KBjcqYx} zio_=h#z7aUK-GLpnc{S?IO_oys9|#^lu1v3NppRB4qABf)K+6~7w*kf&?G}+Dps5yjY&Am`iGD|E>L(O+kFFL5d3zd=D8CoR&!W7|aUu!37_qb{56@1(Z?#r6IT*WC?oZrP7;rciE zA+ZDYpyb&8&L)9p0O^JxvrfkMl4b5fa$J<98yyxM{i_`EmJ27c1YfLu$7~R*qN5kY zM{jO28$`bI=RvBR?~D=o9&3MdWux(KO5-5zhgWN*4N>D+97PPr>AhR*1U00PNFyiV z8Q8A~5Ra`h_Y~4_i7LxJu_B(jpf_y9$MSW_0yb`G16yJ#y6wF8 z5QiLLj-@8uKphxjUHhm`dTIXM?)}7CjH3}k=R5W7c(?d3ivLBGUFBT@Bf>Ui+a9Y< zhPHSE?zAr%uFEog1VkN0%BukbB+ThfS7<;t+Nrx)%AZO#4WZ&i^prrwF8;*&lz3k5 z?^s~O8?EK$_OpF?$!Y1un=`~vfal3qry|Cw2b?w^uamUiXQ_HD?G4G$=s0Va?`mcrl@OC zHYf}~R4snpS7`Ct_9jY(@VwTSbZosQe8uO9p5~@*^0iUdXj(}m`s{Vu2S|7p==eVv zzBFAH40eJpS5>oduNv$H-C(d8bgRK`kWrI+eb)*U$oPw}lY#t2Ao20~J=MpdHo3vn zCg4B%n>#(@4kZD3q2_&kuSyoWV+s7#?psu zAl((MFO@O_;O-sR@AjlA-F~J|dl8$Jb@cAJ3-?phF=6?ZZfwLoR8&sn0Gf z5(|Bz$z%uu78pTi1EYsPg~T}h#sw`sn{+JZJDrPjv7c1HRjq)dNR4NxuDBD`edTMQIvo5kJAT;^NLnAz`dcugkMHQS@Y~8`9e&e7 zDiMO1&7&gwL~lye1Cg&Be@US4>HXi<_sD2!_CR3)JzmIxg*OfSgfo3N{I^g1bUvV< zCDgzu>BE@ge3p*)fjd)1R;en=jDw5!c?km@ti8`gXdfJxxF+?}syHE1N~0^yY&11# zDtc!E+s|pFWv8fi$Yxk4zSN?$MXETwPEJhAEUKu-XMENH$KD$p+ZfsuaAE<1BHCe# zmhPK88~+Sk2edrZDiv;T74_D}2Q10578(#cqsL%`6F1c!v-P4*7q*Nv zs?2#4;hx!G$L7br*AD=Wgu~%>7kRWd0HZ5;rZt}WkuT5S=kFHFS~B%jbs@^rd8$v8 zh5y`kEt}@eJvG=v~$W!h! z;X}#F;8s{CWt!3(%5fmoSvww7^qe6kP?BsdgicJ?j2UpRQ{U60s_+fZ zLs*1SZI%h6f+NM2X;KN7A;D^%Nqeb@Duv=~JAXMqtKs?~)`maxyK<#&qLa~E%?M{3 zujOcn$cnLQaHRn&TCe?S+fMt)Xq|N>ocdNTtsaPrCsOxu=t}1zWQ{hu>C-tdKzponeh4QydSdY{Ep$^R0viy;x7_QG#g?F(okZ<+>=z_H zV!2ul2$rkgl)wC0uI_G025PYDnkOfrtuKwfhL|{N9~ThIqH_iR1opAj!DzctsZ%~l zWdjjmfSgVg5wk`NQ8?Gnzm0l-46KB$_^AU1Xs&lCv_s#TDfl6I?gi$FzMVdsd|CS?BMY@DAsyBsM<5TaO zpp3uJiL0rKBgwjHRLtS4|A3yD{Z%wv#AF?`!VA3{=ttzO&f40+I^!Gp@}lgo0X-=m zoOoGAxwI$@l#V#Fj6;Z8IACG*XF?=JG&z5c3Nt(7vJudlD4;C;Yl2T}+UDq~ zqBs9P$wx+dwo3n#COy(u?StMtU-SOn^ri%U5CYSvr54pvdze88w!&i9YI#pDZEgR$ zzjB}*rs+`IP!Z%#Xd~Ms0AF+sU9GaaqVL|JJzPt9c zKNH72Ec_5I8f>jr)rHGxpesV0CK_lZYPvXnX9qW|GEqa5F2d)v*#~#a7bWmaxZ3eG zf@ePx;9JBWMJd22Nt|T9`TR;3bW_+3-PqRlV8y_N&a(QaB}Vx}hc+@uok*tf_^wE1 zYk`ksO43N?R}Kpu^sFxV8ZRg?38aletG{fl;Qfd6cDdKXXZE{d)$X4pe1qaQZPQ3% z^L(<(m`Hn*A*$5RknnZS?x*?W zbj<9T!Dj-e^TqWs#qN?Hx8DJdu_OxLLs2!0M?{GvgadU5B85qayA)J_lK(Qa`s|7u zBG*)7`}wwB^?JfJ<|Ptrt3#jJOkIy|RfAZ_%b+)U7vv)XBwFRk|~)U}BgtCYe22^VTI>wXs8 zB=C`f3yHx%+Hi%RHNA|3!;fo2%63Ghz(%PO;jB`{^@9)qajS zk{X-Oo3iJM^z~3{Pi9+G&*Az_%#f{aHiRrdmnH%eu{u{pnlV7U#5;WL5io-CSQ{r5 zYDAW9hwfv3h9`1aCjZ{77ECAITJbdv%VTL@??zn`g7SLY>kkNl{M#ly+qTsB#hyR^ zK~H7|?&#nC>5*oByov+n#|v;mnmeYL0F++D4La5>a}qlZy0)s+Jf^QJf_Yp*mzhwq zLTvQsZvpdHVh{6If*$2!5g;ZDv#FXl1Bb|y!{v1#<-@}EL0XjT*+u(qLTeC_p8VO& zuyU8~r^IYrp+=*MD|T~u+B+=UVD5(Nf5$+86@!77QRpeEs_#I(oE~RFnUs5}*xyu6 zE^|k_(Mz!Z99u^-b{_}xES$Pf)S3Rwq@k#Dm(%`Q)I4l8{=_9{cxmhX$8<%CZ-qhr ze{@1i>p1Xb`%Dd#Evi{|FHD&Rq)HPvV(3P}#HD2Tv$Q&%;)t3gIQkt6`SR^aA1~u z1VqmLEwkK=>=&h>e(a!r;h+XPs7h43A_hQD_-(XYs? z@0DRqlWewP9Gj=Fo#PD{iy1!Oug02PXw|y4GCClNCH_qjL~(#qlJspfCbcY4WuG zvegexu=nAPZ|ow3q9rzZLwxk7?Bg5j1D1z$`R$l$rmm=r&Is3tR!xTj#cRHF}A zjY-^Tl{U@qe4Jr>o}Vjc#uAvqtMU$g?@OGFy0Y-dcPV(uSQ+^SqiV6gI+;9!rCsOD zG08KmzQpRUrv4dcI7(hk^iuZCf}3T%$Eyi#U4trSQo+iDV#g9~WDWR+?vffenwp#& zOO4HCONc6Ia)sc$)YVnd)S1jepRrAyMJS`QY|CGvnEQsLyrkVA)Wdexu~gJfh1V#P z;<-`c>r>Vdm;6qPTAkMLKFl!&7KQBS{iL~Vuz`G_XdFjqhvPD&Hk)Y|ZZSK5)5Jb^ z7ClUbUTQXXwq45uojfjnrdjP`&xg$dTQ_no8hSkzdPPc$MqPupljuTksfnQ$Wae6H zm*7Vofzr{?F`%=p4WGXX-RUE40u;H0-T4&Z91YhUCd!y_Pt*|2wk|dyBed!{HVmOw zb-F{3vdn^R6f_ze)fQmxvprB0)!h!fRhQ(5@4uUoNB2E-XR$$URKqO>={kl>+cd5wUdwKvymQNLElv;gD3KJ_AqdVoeo$2`A_Hga6g;aoaW4Qi*wF7O@j9j$@e zXzjSfN7@dK1!yfI+*(4f4EYvM6ul4Iz=eGKCx*& zIW{B&MBZXArpv~ zv-V69w5NT~dEfv0|9?I&A2NIIwV%EA+UxS%*YkMA4IH}`@#FL%o}U*1R~Dc1{LVD$uWwxN^%p+O79O779Q@! zA&d+H;kD&J2akM-5eoo&^XbMdA1G8^ehdCv~KPYL&l7Jr?K7GM2>6>P2^ zKG(8`xO_<4&@SUo+Y)H57oUXh%QzA2so~`5bBzJT$Q4fd=OUP=^M_Jr?m`;3*6Q0; zoqMa*Coi5Cs=CBvm%Uv7D()~lQmZb0I`wE(i#C3=P8(t>^a%^ujgL}`ZUvh^Tr{8V zl>5tHk2l`WEBGkrfy^j&cwMCMHGdyn%>Gig=>j}*08*c717<2Pv=vf$YxY3HD>eII zk-W@2MEG*nnFU92zd}hXzouzpv(rJdrrla*j;dBtQz;PZt}tXZn|^nOZ;ea-N+zEaOeZ+(m2ir%`zU3A(7X^?bhyIm+(GZ#DTLRp%*U{dAf-s;YK z?1p8=jNZE6E-Wb~qo;~<@vCMV^_uz{+m=@>+mU+V!qut8mK)$XX=T@rZGLNZOfv&n zYq>?x>6^dXt0z3P7+ZyYb>3pTCRXs!tUl)zA;?3E6_MyQ_QQT(5t4{tESXI zRgL0*85QxpDl<;mI;J2u*<}`rzZ)ufPBrXVN1T33b^aB7A0I|Y1~7WWx1rznDc(!& zytaF$Rps75c$d7~8?^bX_qqD5VG^vWN7A#hjStkAGb@)zeII%DM1_QjCdjWwKCPec zPz~-tVW%tnz!mC1;dWQp;|g`4kmvNcGW7DeCF+ePcz{OzTxIqP9N>aeXjkAY!fP}E zxC#~Fl|l>jnwxNCit|cYuEMrP%bs)jaZT$H9AEGybWG60Jnt;W+o0;D= zWKjl*7&+)S|6F^ppIaLy+1p3x?XH-KUxNvR zeJtNkSGDdXN1xiapoOez()P(<^bz_aSosJq3Ql;0J_r^)lKP-3ss&zE$@+S$Y69!) z^{OH*I~CJ7Z`}i(S7ggBQns_6lgx4pG@lOH0_I*=EAg5tc&Cn(OF@C3zqMNbg!tB@Ql^@JQ=HQ-%8BoBMn z7cE$0_SzBI42m>U&mV3wdj|((_V&*qjsLFLr17ywBaN2^r1Q$YCjN%UoRsUq9Gi3FO#I8@Jeruu()mdN@wu#yLQbp>)RtG5 za$%2$IpJ|*&d%gn5C!rj4=RHG6+H7}-d({fVPR;QVry}2FI=jY!gm$ei=90Bpvs7N zrHZQ3k$P&U*H!Z5sfRIr zC>#|z!~c4q`3*`5*2>0%{hXy$gnrRAbrCVfFm{u+dT`*nQ)KoA-CuteC(@^>zXI3) zfU88Haqpl&(?a3;Wd5CW(6SS=vl4Y{3Y(eLPI#=3Rg}FSXqsX$xqrp;pgkv&@PA<6 zR2ohmaiv;ToBQX!IK6=;ah@(#Z+wL!oZc{-0!xj|H>^RwZj(v;e|Ee2wejf!-VzRk zdcuhz8h4Wk_*~_`lvR!8H{^L$UZ43+ky zv>;RZ~+ z=I+A;iBDWIbfN=z)uxRBe1wj_no!^Xb1_;{fA4s^=c93Of@Y3FCSt6K4&A301e#1rQY`AOZapxkO{=+2qt*2p zu!FVDdVj$K0Fj9yu;6!gLN*3zlB0=h3ovph4k|mJB|orGaXH+y*{lQRtS1kTwMK6J0w<&}x`hAq$tLqUjwSS?Ps1M%L+$oo!R48jf3-#6C zBN~tcWpH6~0U&@Fo*1kP2?U3J(+aIRWG)bs`eHl=#^}(VaBF9wG+9fiQ7gWaW8D!l zKDtQVE#03YQ-nRLtQ#VWf~ z>_C(>rx8%sEasTW*VwomB<9LuWt#)Zukj&~Gqpn0WAvD&|c6C`@KIiHp=KPB8qdfYDH@bR?xzB!9Z{LWf9d?Q6&`x`9v}_bS zcR->CfzgSxKeWZ}lyUCGI%gMbd)SRl9SFZ&>!r8*^o6ea592TOf3W^(LeJ>6I}|%O z{ak&!lDIP7fd|r5=4MajV>#a_ce{idW`dgQjq98tu16CfR5qmkv(BdlHLiu=ye}q%|{h7`!H)VX%-J=5Zt?NmvHHu3i+NTG+z~gX1&v0j3ay;5R`#Gl|sZ z=$+OMh;{Uw^wd*u7S>8l=czdgf~1DBP^!`^DUI|+N5-oUG&cjuCt!ILJQicpVin9k zARcH5Kkyl4MQOg|YlDn033Grinf7V;hB@#VUA25DfWZvlkq+=g2l!n9&K2O-9pKxa zP1pE$pVhdV12v8kV50!NREkez03UaNs~zC+0&EhX#*kOvk^#KQ0p9BXXJgP`p%L1mwifjZtnjc`z9s&2lZ z&IW4FXVTyMVU~UGVGiokf=UVs^Fa364Ak=us)GZ)GFVUxfWjIxcBuHj>_hmorF25# zFlcA<%p4{l%S~B=&bSax;cW^OE!4#U9XeFEpCQmFw?=RgD?101$G182vhMy&5RKPp zo8n!ZDrs4)Yz$VLCj_0ijH{o5c00IA3~B6vBj|?OH@6TxKkz^|M5>j)lqK(8#|^M+ zB=J}~?Sm#-)O?V&n%j>6o$nVrJI*LQ_)Qtn#+3_s(ZOODLAnHMHLkLT6R?D%-<%s* z6( zEo+Yu`4m(cNGhDa8223ZI_}f5jxX0noQ`nq>ToT)_aw)HtBacn8Q>i0#1k#%L)D(r zz=H4lv$D!wl>lgBV@hKFu2>{_IAWrc{)=Y8lHEa%X1cOmpU6z_2SMn91HH$1E>gl*na zY&Yk`+$SKXQZd)THY+N$05Ct(RrDafmg0HDxt1gM+pVbvJQ;rHm>QTb*aqCa``Q~x zSmNXe?GxI*jW3G$dSZ!$=1SYIXBGH;=Ov?gHPOU%DnV0a4z?Nn2QiF;(Rqv8md{!e zC3Q*bOj4KVKw*n3M4_ib9VqN@g&nR?2dNi$IybJO+;}=f+wa6>_=B2(XUL|01kt?|DoPCHo!wS%Rx7>_zRLB8n_iHc_EfO z9b2%*Fd3awBb=j-5`%0bnG9V=A&95@i~ZN{e>&d-O)ue6-EYpb4joF8By@!AFW!7d zAUb5aACsBx+F_Mh@0t1TkkC|RLD$4b>j!4?&{Y`HCKn>Sz25JKuH^5+Wan3chT$!3 z&~e-C>|whh6&HrAZf}B9pe)Cw`N^M>o0N+Nbu>B}5 z&=es}7fzdDvy4@w?sJ_`4nao}+=Ubnt@&|ISASe{lwWf`h-*!LJwmcOCq?-u=Cue{=AsW#C6R_-{D) zYX$#32mfoqCmsC3g7k;)-)$d$goFRS;4g6S(*%E!gMTgq|Ez<5g9G2Y6DOZ#7drSe z1^)#HzbFHLmxKS4ga4!8FLLll3I4!7``*hk@KYT8EU*5ffS>5#Hy;FkwS)g=27a`I zk2(0&g0FS(4+;Km2Y+M+KG(sY;NV~U4DhQR{9M6b3cNJRnf2W-e*c{9;s(rvD4fGI zaMAaFEhSZ)%?Vcql@Idlr2j8Kfu~RdSlIhH0KoS`io_Nw{({|E6KMW-DYMX2!U(p6 z%f^Kv>w+~mJamVNylC=>(eDo?j=q0wpn0M}M|jH>Fq3sqI+)sMa@eCl#A@?@AqC4& zr}N7*ou71dId#LjN_bGh=`Je>G#{szn4a={8sxYltXHHA>y;iZaeg7hab2|Rd>Zr) zC9yJ+=^?NMnore}Wh(;BpWy_1m#Zt5INJ;Q|0#;)*^Kw}H9=O(i#*L~0jE*{a$3mg zlkT*b(}y|j=hLMA?PI{jz=#KSSpZg$1)x2X1)x^Cjx8bImQWd4fn%>!l)XOdiAbVR zWCfH`C|NTYuH6QU`X@IWFtru&_k{Yk6XByHR*T;)GAe=kCl!rP?axD|R@~@tV1XIA zYgdT4&_A8gl6DJZD0N@Sny9Z`H{k!^H4rJb$CC9mauH^{iE{Wt_vbZz`sNS#N)!%n zROKBJk}eN4#L(yXQkCPT(!B^su-(-y=u7CA13}3_C$06!rzmj z4I4zWeD4Lc-y!vCwf~pcTl~L--^IJP{@t!+#)WvjHCMe=QTB2m837eX;@vTFBi6nZ zu6<2|RQ7bBsTu??Jx*q<$k6$&h9?*)dC8O;rm{?4&amOd+g|+z6ToQTvOv@#r}xMos8cX2~T@S{}RcW{w1*T z`f5dQUSB<2@s>jt=2^w`ufRaA5A+ zW`Y8vgiAiet8TDUBpTAfbD$DwB3aaI3!LY3SUo25mPeDLZ?RIcgF9gZ*F@y+GyonN z7bC`Ic8rPpvISqjiBPodFAef(a`rvG-+17C93xLwV`_h(>1=DZ@ zs-YP_o|6IVPxu5wCG#CaCG#CaCG-6W&3Ch3s#sP`p|0U(za(7|_$2-`Hk!H{7Ycto zs`>q>c8Qc7V16rWGd$Sg!8Fg4^RDqc$8ms%xffrUWv{>kq%~}t0%^cE-!Z@)8~~h& zS#sH{0GfvFu?*BQ2eryUorVc++3SLmhV7ON)J+cRUI*2rm%b(_Y1l5$KwavfW;>{_ zO2d|Ws_~&)m^_`&?R4x(AAK3XpDba{8-Mb7=qf@7NjJjcOU}3gMcF&}+{6;+vHRg1 z6lHI66-k_jHtrk<`&zsA1IYOIQM;|sJ%U9g#*`)o!xb;x#`mA$Cl67B-Ps>bT%+9( zPjPQ_H}`7u=c3Sv4qan1s?!nIQN9jVjAS);FW(|FCKth&Q068@s@T>YYxY?3bbK5%<)y%M#k?*{T7ky`Td zR3J8CD|9yF8EMtt)U;}Eaz$cDTkGu0(MFOtJ(l>Mk}a*Bxa+{EZ)NX%b8uGb!z)*k zzP>eeVET^2i8~JzPwOa*CQrDeYj76Q&g|S3sRNe}yKT$htckn+UVNoXbi&icGuzv? zTsmlF%fwwD7MHhFBxV;^fZN)(%qna3rS?+a-l^^H>V%HOmNZ5Rua?Bo^urkK02w1Z zHb#u4|An^Jy|dZd+8s@P;|5+Es~!DIV-?K|N2aM@IG$frcaBDEPdB>_Oe-2mJR(BN z#}H@YftCC{!yl|qyEn?IgV)|w)Bp%6ts%JFup#gIDi>y|!YwWmIIAVx*g|HfjvW!$ z#_Rve&4^>0*K(Q+eUp?<2T9vR)xLJ7LhNbsRwd=*&^WpXmF6VQVoqlePK+*+4dEuW z%Vl~R4|j*pH4IZ~?@JSk!fjjj9s|AgDP!r0!tUHE##fwFFwlD0Ad{g}>!%>ex=UDHK6fe;j}NRO)#3T#C%;bd1rg76yLg-_Pj5Tp}n26*T$@I7;~TaMA1Z1 z(by6am>vl4c~Ksrjc*O98xu?B9#0+TB~M?bT{sHbhoW=sdC6R^o#z%={yhE9oU1PL z-`FqdF>m`l;;Ya!k!+|E)t z*_2vkj(>o84=sF4nt;$vZ`nnsyTEIYJM2i%9lZ9qkUt+6c*7_tN`(+ z=ghu5^vGfzqK~xbQ29uQ4#OYe_uA0Ix^->zVcoj6@?qV&cKE}X$nbmGQ{NiF8s3OG zG7DVFo4fwR)bXYWn2!aMN-r2@h*r#M6*Nq@s5{5(c5V zVlWb^u3z$wO^YjlWS1d}zcJ=J-^x{2#%mHZ;V$u4o&woCtf2ed=E_BjbqFnL(P7e}4jpnAg&SAkg!Wo}6S7m7E@~P@-;ycS<-t+m#mtV^E0F}VAaTD@J~vNH{aO)US%F=x9jX#j24N^kq@*oPRSKfXSv6_W!G>=( zRZbx}K|xwSiyXNJ)EYw-Zeaw@tv&tsj0boXbXZ!U%%cRwQ?l=rA;W0NhVVBU7^Fbs z!~7~}0a|Vf3AnxIJ#%MjUtZ{LTtZfaNi~XpbnlR9*VzV2@ZHtH z(4w~-fV+_YbLeh!W)3Zq|8wYWb7l@LlK*q)?py~Z|L4%%L3bes=+NEy?m`aGp}Pxo z;a_CuME4LZi|Md0Mm zp+PxzlJ?c8V$LeDqGfPOrRRq-_3=nS>Ab_AQ;@DH;F)mM zw&sm63cndG+Yo48tpW)HWfkz>Pugt+hojxM`|MTc*tf@O{qCbNxyu7=j`z_p zhpq0aYaNec|J9o@^n0&@3+;@Sbq1Of170=DZl8mU5;-FmK|rrYjL=s-k3zJuk0lE< zT?MA)+IUpe-86%Ot|U+B%d7#--?RG0=*ynR#vW)Ms{;G7F?`wAR6QwaMWSBq%%9}T zs@S7l>b<>zrXQJ5a&8dbu69kh>^lt7i-3#Q9TaHZ!KoxKzT>>0ek3HE=g7gSJSnn3 z06)Z<8-#k`$-zQKhfL6Nm=>OmP=_iB(d@n`IFoLdu zB1LOQ-_G|C4(w@+MvXl=)6P8v-~W2nm;ILvMAp}}&8e#*sp*`VdH(;1G`11Ed`mrP z)L$o(a^0eV&yNc{GU5_Tf_gu{_7q6^KV+6F>!{zj*y@D-87xhRYgDYfJ$X*8UmLb_4zomIf= z&I;;BQq@_|lE8w?c}!#X@AmjFO)!C)pw)9^1rK1eb2@_lgzh_10^^(Hf*A=Z;bMVK zm^?;cp3y~4q+~@nISygPUlfz7GC9sf6Pf;E%E`5_``?T{${yx}y@uYytGpa&dQ~$W zicxxY?eN9)&jduEZx5C9;1z~0Fs6{jkRAn)?c*2jfxi4L%f^1GwhwTj*rw2-w@~3B1Y9VBpv(Z) zUfGL7kFo!OeGHFv_Ir%7In$~_OKr14h+77@T@azAwpk(c7@HyV^bS{OyA^b0g&(-W zAXj;z$87gR=&>GGnC}X0_eAKi{jRXU7256z|6{IfxXS#G5&VTYPZsQRX?xf05)sn= z24QB61^n3U$v^kZ42B+*B{Ud)P?peO<%6<>28Tb$`l$+$8k@zW)Yx-Ke@$xaIg_fC z8v7pjAbTv&J(lMl%NzcnWQADS`}MyiK&0p&u073HivHz^{(Sm2k?d(9}~ zw>Ur%KgUv9b`}(|Yokqy>>(MzYzKIZ13ZBBZ`ou4{>1@q+@G%T^;fONcQ{bv`vMFL zaJ~clO9t?NIKVasxJrOO7NF;YH$MaTa|d{*1N^)IBLeh%@M<%FKXic89NlwgN4lv>XeNw>PCcsWDu|#%W1~A|N3mxF952$gR06ibP&X3Yxz4aCQ>OCCz z>U#wE?*jCE@Se;7{>=eC=K$YP8!G`cvFi2=)Gr;>5(l+GP~R1l#H#5Ts4E=QTnBZJ zpso^>#Hv^Z>Kq3(!9jglO}iQ>eDGQ~!Swv`4bu>2)@MKdXU=wQpfUa63-A+-V75T> zF3$Q{-~@3fAAK2Y$4@|tJH&*X@e^2%Kn`b(vt`%&f=<2N7H~*s!V`^gMa9Bnqg)t- z@3{x0(7VD;uh!{&J0)3F*6b$D^a3kiA2gGsw44!etV9@Ion)@9iZ?G*uP94a0!{aSLm9}le$RQDR5!aZ)B&gDuN zMy<4gqrx(A`k)o9k%vo%P8&BL4DZ2#-)+7gi65B+hvM!rBCwh9%Xluy58bSN&M8fP zfLLRyv_Wn$)ybZMUW$mWUlp1#UwwS2y$qFbH2O+x1iN&c5gv>YQFj}7RNu1Lj=0dP zr82{kt!iX?KXQ+5>iu#>fwW*OXgB+3)Q6;_*A@htzGFRUCR6PbFjXdImeU96HL8Ka z)fx9-%63pDl?=j{Hc`+I^Q>gHKd2UU>XGEkulf~1ivj9f{wg&4Nd|zfI{;U;oHL@x zbZ{VikA@tRY%x|DHN>jK=}T3Hlf!f5@unBbJGwzf3q9-kqx)8zn=JYprA!sKsa^(H z-=2}Xqw&q1XY*e4TbGH--GlPzF>`!|hg{I1p5{$cHAwDFA)nt`8!g!k+vP$lOOqtm z#9kPW1F!7If0^gf^rYhP9D8p%+L@RBgQZibHfv7FYp;c)e5BhscEiH;n;%VH;K-*G z!mu&bBZf`=Y!lal^LQ@bN~cM=?Qz1fNK#MJV>Bg_tSmyMKSkE=`9-!XDLJ}?#*U}4 z`o2=AB2t{wqn(Df@Nt)WR1V3S3+ppX%y zA20JSP3@f>I(0v^c2JbfEyaNarQXPu^XOnbx~+5d2KuYO%5l_$3o=t=u~K+8?j#Zx zJFs{8*Z~u`MX5qzPkMxP3MZ~Zh2CnvtK)mvSS(N2((s-)&rSN@p$oH6X|C20J*G79 zSFS=;f8g>H7H~wJQGN2?+L6)X?{|Mz-sf6p?BKNZ7xQhA37@NQAuOokh;KE~4P#~R+C$S% zBu*(22=TC=o=;7-%;Gz{eoI}(KdBLgG_{+*U3PaMKR$8Si^b)8Cr(>kT;3|K-i1Qh zKW@#$qh^)dlsm*SDo*_NtW$3Cf6HKd1Vur~Z~Wh!xQoa73Ua@hdU5^s)avygFsFLz z@6}u=#4fBsFNp69G`-2}vDf#_M}m_^&{7Nro2y?+v+CHhu$|A|8u7i(coB4pi+5wb zH*l>O`Hu9lmyIZrR`ybG3|+BaaTm+ptKS+ad6z(A;dqKYVz80Zd6(IrVbDou+<9Q#K;sPV9yh75x06jjdApWdDl+(ESpYK5T0XMHLIHPk`EwxT0cIb!EA_&d=0!o~x_I)m5OnM!C9DFH_fp)YXsQYurBB zn7vn_FrKs8MLJDf%w)o#nasVLfy4C(RY!;$itvsR6EC)M|&B)=?b2wpr5LE@*ylP8*iPHS*x7PBJq|!KeKk;+ZytPAMHcrIm zx5-!edaPuBD890B6HX`?6C)+hRG{To9bw$f!cb>UF-^Xxi8ZU0DF|43v}858?v7|( z4zWThA5NZdZZyEs%{~hoyIcaRx{oGKlcFav!dwQL0!BJb^+1=50`;~VS{^d}7*bsJ z6mRIhcp&{aH(3`T$AfM;dA)Vw4<;9E5cFudsslba4O2@`)Xe1Y-; zDI0Q&5&{*#701KAz2@(``3J+Di|?L#B2~dUMiW2gOL3{Aah>UKlSVr9NicLPnWRL< z3i!`X)}7{0y>h`@3b%P&TK)#1Ul(a?85EP|^j=_vwd>~WPp!G&9pbw_HEnpR{es;T z=G9$i!v5)VR6IS^{Y~X3S~cPQwkKN+vS2vz(Z7o@b~2_-r%naF>#Fa=Kc zDsq?E(>Dl_#Fay<9#Ec*cv}$j45(^ZR-Q@yt|)V!eTL}r5b+~`xwV+m%ZEY$i2y7bdubN31B`d6j>h-y~YF%>e+*k3elx^N7|F`qL zY=Y#DX=?=14_xC@Jv_cW0&b~yQN7MDFnf0UM?Il47CCj&4b5HJonNdKn0;s*BHkkk zb-cNOQYlgpvr44{QYu{nukuS-!|UqR_|Siux^UmrAz~<85le=yi43JvbB9JF&mpEX z7_=`tfgKE06wfr@G}XW4Q^vDq2ihp{^?1>=|2a!GSv0MBHmz{RuYl#Oa-zK#1ciuD z4*e^0(|g6Lv#R=1A&!NFYct9wK&c~v;;b+>^%N)i2PKomry9S-zxFQmwqN|jX*olq z*-r*av#1Ku%vuz0Rrg!A?vLpJ^xzZ!?MF#a@k$oy)5^}Z!rM!54K1hA%g6XnN^5#f zW&Q)iJLy&Tz*w599_KgyQ=9l=t&7m^c?^%!VV~USm9>}qyaVxetTfBC{HRj#Hu<%Q zvdbQt-Oc||%hDeZ+X6{?rdP@RKUP^<4!6{UOsh*M8_HOkpcI(*hEX=`-GZ{=$C-iR zVnj04Fq`Tsn`)m@{YRR2XP1mizGhO)qY*ck6s0)({I`hfkZ zDuDz3dDz{3T_rG4s@Z?95=fq3D89mKJkXcvbf>+@cH<)g^Rjx$1B|Rx@qCLfE1-BO z$5s#x^zJ`yeAsO9E6EaQ9Ez_gzIK@*;9LBg2-bS_^2dYorADk+)_i`Vg+4+YR;o@X zs!dfE6Idc(*83y^$skGE96|kDM^Gv7+5Z-)V@pVj(2B;KTTYicrMnXaGtiZEp*~q5 z5#8PJBxyKsJ-w^!G;?n0ID2lXG)?ZXMPh}XJpsHh!cn%K%4S?obEUgv6zxmkyZ4If zgp+fr9s<&d>OgzIc_c%14mPvr!ZvN$_(7Ei$_flnbrg9H6ItVRn$Dm&!sQ7uq2_O+qU*v z8=UXL5qicFej-b7hn_QPiM*qaSu$2W)ECP?`eK4Rjap1_r!kAi;-e~0KB`uji|6VE z9qc0xu9bs(2DD;F`?Awo>ZeM)jD;m`hUYZZI8Cd{_i~!AedzWmIjw!E*Lk7a+4Dv% z%y36}89cLiX7SACnawkYCwKiV?98(>&s?6lJiGGjYJG|{l)n(i3MccPn#Alr@hvmE z^LX^C78tu|1^__}@!9Femt44Jcbnpa$TRgiY-B+FD^Rcm&qniVg}2RwgZ%h??B7S8 zL1sN9JY0qi0Ya7;7Bh@LCaW22f6_cVzKpnZ5N2zouVNv$c_NqATqd^eD;X#jL9ymb zZRMOZKTnSuzwHgxZ@u~En1NR~`ulJnaelHKB){L;~RMOox1QQg6Jw*A%? ztu5MHsEQ&5a#q^*h83!W0|}8DwlMB$qNZYL?^aZ7Yg^HnF*R!t%9jTe%3TM8dcDV| z9k1bDM(!{-V?$7C?Yv*)l;q~sKKP4H8NVLxmE?9G=9X;8Y7y%ww&d3tiXNe82EjDR zW}afBvlKl;(YYznmATUC+7#JgJfwx^EmN~f#>lGwLzHr5u%Guf`=Wi)Wv0`^Wz zF}K7#D~a#AF9lWxo2=dT<=J;gEspE#%n$HnQ}<=3_WoDT%m316vYqZUFe@CrLmKmN z^sYtb@mmQ6;pp>{Jn&I>^K$*Hl+Sm)tkBQuy4a)@wf`*dTg{~qgB#=^f872f1;5IM zgG;-4wEK>lar=(+y)yF#)p?}Tz?@?5jz=hvgQA9KhDsaUp*?GBelhzF$u@O|rc`em zw|`b{|IBU+$#H1Iz}`BQQc$@z&+HnL+uw^$kaU7EW& zhHo2MTl3l3roXyQ=M9q>XJpEdQ(F^1`wqdt`WYuE*8~Y3$sin|kxH;h265r$tLhtF zR9}iKn%WqXeSC<1S?oRW6k!!d+Jqu;osuox6!FeD{blV%_s+N8p-Q z9IpR1>f!FEi8Kp~cJEJe3U=e!Ta?U%=3@5#hP*uXy^H?FuUTCezw#Xlc-rPHaSuJ! z)*nSSljFU!KQOuS61m#6%oWt<#>mvIIn#S4Q};xsx1ueoI%=zOH{IrT+Z3S3^4LzAVqNcR!h+Jx?SAV+og=}p~^yYoe5 z_OH5E)lQHd$OM^rq}Cf!k%XCwd&ZATRGdL$Y<;K*Th&*l47yw(+JD5!p)~)?-;1dl zLk$_tz4-@yUX*XmHeNMf>=8#0!q8;P$ndX(`I#3QP!2{n2C&Y1=$N3Kdzk^{4a)^e zPS++udBp>gg8#QrO!Vd7!)^+@Z@{jxE(Lbo=-W3Bwr_t6u+{#q1=xObUnU zV*3IF3?R0hPk&Y)1F_l98i-wY|1lx9beMtI2Q`9NDoa=>+WZaEu}z$*2FH3*SrTGb z{N_Iau^)8;Vqe|e0%C6&luJPDwEzJFh&{!pe^qY-vB)z9V!i)(Oo-i4Y9MxhwIG&C z`L8w<`J1Kuf7h;lw9nDkQXqB)-TLM!f43YUwsLU`h}~;YE&;Ji?jOLrhu{)7JfY_S|fmo)QqzQ-}3=lAYn9ryG z&iMvnl}{LmP5Rw2A@-_cAa;Z`T=CNElv(*3Ly^B(I(7;xW+3)MDoaA_>V^Lah+T0! z5NkuxDG+DB@;3{y z&siY>u?#9pLhPKo{u2;8<2WF;I@$ta4;qw9Kx}b_;q+V+5JGDW9=s|ZwXW|ZGIUr9!Jy7|n!lYFSg#wOM6QnfEpVFRFYGOypU z8!c}%bE*=$nPc?eLJ2+UGe`2!-}21M^yYZ|Cp;?1cNqj(^rr~lqy57~`J))g*uAEx z{AX(f^LWBn*)9x zzbQT-HQ>tln0T+$04F{)o|hVMLHxov3ZdpQ&WslXpLraJEJr6~W?rajs33#NCzyDf z1v$1?YBT}g)JLHIc`x?emM|=TC%z%2$WUch;{D$V+9%qS=0LhhVrTh1u@C8NO2GWs zTP*@YmA9}95tDURbE;6~)e8J`t3XGAY`M2c`2zw2AlvR$epewsPc1E^kkP3jNAPbE zADkKzR|tw`;1`s$Ss~v`4XIZM3b^Jp&k^F^^Gm_Z!ZTF$I6*P}G^ayrR+dPkeXSAf zYlW&fHr*7{PpUq9X|?*-)@(wd@l0#6AN&Y$c89f7d#8Q~+CtKL1GrSAjpJfGAF5m< zj)-CDT*Y8(IV0jtX{>83;5@E2WJeIp&P<`>-N3yhdqR~cpV2O}rDqO$#?M=16_Vb> ztVq9!Iibq;Nnh$-jzQYh9M~z!%Ha%Fj>&mDIg7n^QRhg-q4d!FX^4BJ-pOI_+AQz* zLEPiy9*B3N{eMp%RJo$8(3_p3yKutp-iW)r*j-c6)ULe5o1InBlv(zEBsw_Dq$}TE zI4mP{_v`U@ljq9`n3mCNZT!{3VcDU@ug9M^+d<9y8x@UhG1BkJjgSKv+}(BGUeCqXKAAc*P4teT^`TDf^k zcvzHO@Pu=$_Tj=eIkdeklD-$uR@=;7H@~5hguQnJm9VV1YIbJ$OBQH`)kQpMiLK#W z{ANMA$Z<3B!v%G%8X)V3h`Yh6zE#&uN9j*Ls?iAyyOb4T?_yVM=ihf`X+W^*D0bDDQJL99*xp@tf~8}WMJL9{N1 z;}7PHsoD4Cq@TOde{{Zjb56v)F)Lj2QARlZ(}+8s{dJBp#Xi7!Hf9esA7@0Oy|LEn zi~@R0cU9wfUGr5{z45E|3APIUR^~*6Lq{M6PH{{DQl)QU;`k+|l1;KaC=!A?wkpmz zO+XQ?P3YKvQ5P6kQGKMM5e$?ti0O;>a{gSP5I@uIM~0)**m}22 z@t(yMvq6-*tK**(d&9CS4rP{aRHE{%LU`LT%s;|4JKLI^^5LFH9=iRHq%JMk z87|xssw|;$-3(WAgqyokkjRO4Wl%P=i+xcNmqJcx&}JP}3k}#T|3sB8b#Q;1_+Fa7 zi8e7_oEhUX)m`+6qXwMen0aqR&?fxl35bY@)Z;1=4aVY>b$f6kBj9*>se#fsR7p>cewCq zsIn9;6p?E!K#~(@6O?(}KI~2=V3Xrr#Q-jYe2+%D6D`_89#debGMkL0as_ZRY3ja; z^t&ymycWM#c7E2U&d>T#rM4|)zh3iIhPj9St2R?J%_L;MVd%NS*3w-*xe+Sv!+FsTi%kDNRX?1r5wRA?}kRl zoRK!%Ejc4?s5|KlAP-3L^HT?9uI3OuP7cbR3Nb3}IJuwzcdld7lgmbbMkJl%;|_Be zu*J5(k`mC>X<|}iUu3e5;jBAzZ(^kIv+`jeiU`6M{~i;^7Lql>af8ny`}qrf&Z`yg zN(!l0H}7v(EaPnTXL3%^PwDsgmy$m2Yh1=sL-`-$4}o~}xdrZp7S3h%$6S6eT-owR zgjRFP`-7QUF?9LlA!C;I`s0|A?Lc+IzTRaZq<`@|+7j*m<0%M<`{$bXOZC3PzF(~O zckTPRqC_cIm#I`6XL68uz|yutpSPQ_jo_F;}VSwCa-bCpSlK%gE?GLE?wJS;W% z^7!=B&%yDDsh=0buhhLA)D}zvVh?i(TLOyO^SM?T~(xNxy7?}-0EpmDV&LCIPr~cyYp^7&su$X%NN8rU)6i`5pWgr|SbBWG1KTp;{?pIE9AeBLSCukCFV}IY=f>@@F-7VPJ z{f%j?YlN9tgbd9Y1i*WfD!n`x3p+e<+W~S1@^ZY6+{aHdw4eI7`_-!7#&k@vpK2kG=F%L;Yv&XWM)b7UAxjf4iz(Q{ zoJNJ7ys8K)_&XJZz4JH&pXC(za%_D|q!?8s6dRPziA2vh@OPODH2MBas*cXguvu|y zGQf-cjTpH%o1{{j>QV~g^B`0)e)2)PGVS&=GdNEn$ONeqW5<990?7fnT5IpVbVBw5 zw6zHQN`Y6!r`-de&+O0ZBrEmMdtf7^#cHd6x8Ju(i+9lkOxo{MaTm@Mm~q5AT_@^K zLbQ`TUL7dG;=XEz__Q|^gRuf<{JmdZi=}w=_6~V*WnYbLL05%Q~yPx8s5845cgyXlkCC>J+TR-ABz~(32yUm>yxI zspbp@iImm5Ezj|$c6YqXdX^OI;nr}ubksWObh6hjEI>n5hbSwib=o4wld%Fr zV@xF+uADV%4&%#acU!Do@2XL~-cZ`o!QGGfW6muYVslAo;h|8)JBo{5v_>YJ70dbP z)r5H`eliZO4s+o081|I3vN4jLeaLC5H};g_ipKW0{InU~j@P@uA{_T2R)=g`;JMa`l#{H@aKa*j;aQx~#FM-g@e-uioSh97*tS`J(`nOyaib zl957{`7l{VBvknwaa%j6Pu<2s`gp#J-Mz&XTUpF+j1+K7&x#a$sC{tuTcnmxzlnqza#vAO1uD~t_G zr10(Xzmy7}bfI@_I0K_qu(*Y>?ED7}L&?;=8PjEt!+DF@zkpX~LNVgq_>pn%$r3hB zbSnCfTmov^>w7klDp6jh|8o@vVHpX5xo(mChxzSqH9=R!h7s42|2W*u*vp&vPN^OAoX>GQ;Gd<2-?D^SteMUf0qZ2LqeEJtS@(ESkC-lcYDGLcBE35@G z>u5&4Sl?Q$>r>HCB@>$yyVZ=vF(BU7i8>q>U+Jv*#EH&ma@Oo|Dpt^P+7pRpgoi~l zDk9PBkyyTZVRvo(4A~Qxx?gY%3)Lo|ni0(0Mcr<^)0B_P`Sp(cjf6f}_^>}QU+vCsjx?Om5)_KD* z=5I3`){>5&l$*FPDJSw6!hDZufP%N_ub_H}FHP8gK7!;BF5FOlQZYK2V)uruQm<#J z+e=b-PO1BTxaQNm(!vd)yGj5;ZOyGWw}yXd^k$CC?7D=Ig+Sz)MG-tHK99> zC!-VX@fiJ@hF6vWZsZvtTZ(!h`x#nNTDX4BOR*TEg6H{;v(ADH;PYWj@zutPZ`mS{ zdZe4N*qsR?1F6Mu+e6v2H#?p|@4+|;1LcN&E(9F*6)K?5wJ0js>~Pw&rm2ndVq%tX zjPEd6zdiG2CwhghT)ZR_EjgJMH$wm7=QPhh>cd1iJlGqvIBT#w#sj{c7Z1Y*KJ2cE z6uv&I=kVyL4sC|=lRiA!b)<$VaxdBlz0Np~*Xb`u80>!p_TjzO79FThFYg_R4&hvY zf61lMneC4{Ynnv0yTMvUvM@a{9dsEv*o#zTmAH|Ys1d|*GhUK1qh?Q@JVe~}IS{Hl zbo1Mh=m1nMHQ_a};nB`v)o!OakX}QQVbRWu1+qcjPbiBnlzJzkC0QJCSHg+Xr&nV( z{wV2^{cRyXw5C@+w8oqQIU0dpIhI~It?8A{w6~^LPbg`#1C|9L@f%rtmOB zu=F#US!iwF&Oz=UhDB3~8 za*7QRBQ7i|R+hkjRiOOCM8L?8eh7*M1uJ%kfv2QC0TPyDNSG)bG^{U8NXk#GYxasw z#k$49*6WI-iMAS=hCd$Dfnlay30#gLaH4r` zPHej7Xv>L*i0()nN}Y}IHZA;(Q83crUI$N6z?_pKjk%#H3uB}y#XvPGUW4IjrM)-o zz89|fvQ0DB;>h2oaCB%>I6Cks%|p%7BtCN+fQW1Ue0S1(Lowz9ewt$#MB&CzMGu-J zpbrt64au0c!+bDL&f*@yu-hIrT(LVO3!-4?wJ8s-8umubhi5FLw2b+Vo3T(n`U=08 z#Z?(&I^Yv=QU|V;&bX=$YC9ilOCuYfU>OuwO5?`zPQ?aW27snu zOPf-uz&_>z?L5?(p}XX1of-dDOMD4OuSUAr?5v3~&BIhTYDqw7+ZaNdu>{ft1fb?6 z?xZC&ml>R~g!`#<%h>d46dkcm;D~^-w|O{f$Gvo*8v4{W3`+~UE1)l>(b0f6Zde*s zq1L15&SCVfOG@393H@iO3(5Z)$Sphj9PJ5NMxr-bLQ6}Qz|A(67QQq4C#CNC_*ur@ zn)o^Jj+@dv8Vf&rk`{Mfa;e+-NlGp)+%oGt`g1P)Z05NDxW`BV_(`#|IfCz#2EH3x zz_&WRyh|wpXO4FnBl%;>J1OjP&*a+);DQBr4GMs}C>(Hu(&!{{13ClrQo;Hi1J=$< zhDR@&!&b-ux8ya7Y;&ob@fyBJ#J*mR9oD2{VCBxI;!xY-(65&j=Ph&G&*G<}KLvd! zl(@a;m$);NC|R*A+bMh}R3U{bwG9lAai`u{Y~jBj?Ot1O5kU-(USZh4pvw2ko^FLP zRn@bv5LbS?)U5@@I3FX?2LzE!5&>L>K|E;P^6>4$|IbmQT$yN*_; zPp_q|X0F68rO%WY)0T&h*X`{Xel@{JXPuG4p08WPxzr+N1AJ?tnQlywio@Bn(Y7ZU zH0ul)^zdc|y`KYbaMLgj<8a`xiVWYLd3`tOn$p76v%Vj(^RJ`>G^4ATfA@nPU5kVs_h(FJkT%6%Y^K*3>lYuW zfgw{LgeP_h0P6rlJ`^r&DDS4;43Czi!P>Kn26KvyNyN`EFPr+Z7lvWH9D&tJ=B~ED zMxbZ$#9xwLGk;k7uMDsGBq1G)x<8npmOppBe>k+Q7-jJQcg%d~&Uc2nUFVm&tR@)) z=4W;65zr~YpPKpeh_L$wYlZN4sqXrEgeY(L1 zvz|CCS44j&+$<@M%vJuoR+uHl9$55IzLDWF5dU!-n>Ezyyc;~sSinzbEf7{a-t0y` zvX@TdjO2MMr=0?yjQ`A(&o}%x?0rk*68zIQU(-$wyB9m|j~mTxfT{3tPY5$9r8`$M zlzt_p1?h)1TJ=N|l^FkU z;fnI(OTFw3rEc~rO&Zh^cZpw9m&tg0OZ+R1UWq&&M;;&ZZKlzeRQ;hT`o+*4MP#EH zt4iI?;leNHtTUH+nC#8)Lbe?0c4kh-=-&hM92tUob)a$z?-xf5M!FWM*Z}{s4;9Xym(LAjEQH zmVJ?eeRiF|2G*fYGbglO8APUL&>)r$F>5%VPv;9t+2H}|?lyg6Bs!Y|9thFHW;YK9N{H|&pm znX*4}_RqCH6828h{!2$RDV&NczBYCe!R9UH@E`Q~|6cye8;)82GX($3@^NSzosNEN z4jZ{mW#37eUlH#Te+F4mqoVHO{4(1-F#h1KlRsW>y=^cUo!Tj?Vp&gz-X3TBbV+la ziTU<_@6X|h$L!BP3H}%TvFFQ>oNN64SX>_Q*Z?o6yEJ;y zbIiju|7l8;9~Wc;+N0 ziFC<-yqtZfORqyGmgHa3ml=f^aL)8kqY#0Ok9JqZa@3_3!(U{^pVM+9Y_=&%ndx8Q zQn&X^9^^6Nz0_S)lHJp)0=*}bO^w~a(y6ady#Ba_X@-SJ7Xh+< z;EW)2gyEbiI<>`*ITHb#GDLU{hW&L7H3=xPx20n0_#yk!@7Op61aYgAaZM9U-xUzVRVU;AY2y?S z#66XatG96q2;v?{#yw`^6cEHMO2#d;aS90H<|gB&+BgLSaZ{3USJ*fO1aV`Nas6zZ z0)jXv8Mg%_hs5+u2CT9HImv)mY=AtxT2OYaG+Fj01K!gNsjpCg{_GVl7{dkrIY#)$ zjb&r?XV(V)UJSl_-5mDNKQh{g5Ja(LL{u z0W0#s>~b zcltMwH{Mnou+iJ5`29FjsSYpxUd#S{_5;4_p)QKXKR;+{{C(83EK-a>3_u_?V#`pJ z-gD=S#+_lwh8Fa|nb)q!FpU{b1g!wt-5kedNm4zQJmuS#asN(YFu3Ic*<@;#-<`pd zTorqb!Dz`)?f#Lx4Rm+kVriEkUF>P2lyba_{Wy7gt*F`+2@N{bYlXJ>ep6GkEu@VH z%I;-vD&wd*YHj~oa1#oLW)4pC82!;6ZC9lW7ak0U1~6dt_7r#~FsrIAKclE}S6LTW zh0<{90oJGbh`ETtXMavd>%|4$x=mc5uRm*y;R1uNrbiqq#GW2tS}rgEjG?&lr{lF` zxQ`56S?fn^hAuY4`D751iET7j5Nl6XdxP24N{6{#7n|)v9@Mk+ZrLJWkZNmM`ax4u z(Q~MW$`rE2-$eFa7zFm2Y%1-NqX*$ROeJIUqMTTHh+CF-R2xqghJ2T3zXxL@AnXX9z6|zI}G}VQJgreQwmp>_Yjlm;IE`T49IE5 zE^-emdiPyGH!&<^-B%J~JstgLj(Xik9{09>HLTQ~ERI zKaemvnRkHg!Nmb=nRPe=g`HpdeoOG)3a%LN=KMlq+U4n6EqKLG<3nx5|9zFI7)=00 zPEQm`VcD$;>yapInM(1xOb7~_KsZeF(qG9#kpE+lu=U85b@b zX;&(HxAL%fe8jszba!y6HyZ-nBvJ7ad`D;b=KvWIoMjXdt{-g3UN|~C%in*h7TZ5G zP}-)jslq54nd9rW9uv=E-{I!GcYYoXo=@v|2d zEmDo$d9m_k+0s_BEBnrX z(DRk2Z{>BE_wwm`d1|#bKK2~gpAr(P)c!&7+&O*I%GzVlG~~)S1JPvQ9roJu`f46| zn@03x-*3PITjGG_m8Ywgbk*CUuqzUUEhx!vSJrJ%==nKU4yZ3VJ@-tKN4y(+8hDPf zrj<{P_f?g(heEfCZ2DQwx*+agrJq|8&}dJbSryXO|ZYzGQ$6{t`+=eJ#&j6KQ&{ z19(J%8Wj-?<-*jtpP&C@k{G`9J5#<4*7_MgD5uy|9Uy9XAiQBW*OZGmV0YJN%w-}^ zU52@EnrP;etQ%e!O9Am;Sh{oW6{>le|MBI9FFm1JqJ4fvGGkGMJ3rmPJb$C5Xu1@8 zi~lGXZRlB%sin)dGE2|45f10tu1{0<){&;}wF^w$BUSg0say03VEDz@@h|(4WtIAU}ISc_6(u<=h1q9iy1A9?aEQ3-URQa01QV^ zS5WI#sVl+-UzBw>9s_Zw|Av18a7LG{OY4T}a6nt6M~o%M=P-pdgkvz;gyw_(6WGj6 z-xYBSbVviuS5sM$+GW19rc3C1aA}hz?NBLW?Bs;qQZ(G$hg*;D9R5%%=OW9YA!{sO zuMaKF3sq1eKY4RPwg4Q?Q@|j6JIOUqaC}CY*n1@7!SZ*kh`&NeDZ3cQMSF9jR5Pd^ zGiuadD+DEH865Dq5|)vF83pp}yG)(0_rJ}DdYR?Vp(Ot>10s}fVv`p(u-A`q1?s$`#h(pjrp%8D(;C6P$?VUe$th3bO zAx7ukNe(AkbaVs1wDlXvPVvDdKT(}%G+v}vqK(oElCRA<% zprE=jN7wv}cxd0flVY-()fHm}uj}dy4#kwZJ4*`=PAxGLF6a=JC_3 zy40-?7i>f??5KR`K-o!*nj?GP!+xo0BY~z0noB!n8r+9)b(d@JRJ6JgE;v{Qe(Y4X zZxKxtmqTdjv&X-~pnzzMY=waYy$q)>kGNBXL8%5bG`hK+Ww>{zXPl_`q-C<$bDD|K z{+nlkY=t8cY7+Qu^(P5&CaLiH8EHhejtD2Ma6*Zw;IH`?`a#Pd&_W}gG?j-T_Sn<* z3^Ok?{iPJ5c{DhWxi>IRS0+Y>p{K1W>&q%GUYM$X5tCj(df^C`fd63MK8$KR935hG zUt9efiHu_(1XXH}{!@xEhp`Mtr(q5;7=Y>C@E>ExeOa#f)9DdeTzf@E;_h%5!=S*! z?ZG(5!VYqsQpQHjcTMWS-4WQbKq|^Ufu^=&sQN~{@kMIWx7TIl1R2b}SbBqxFmP5#4trm$QVUCHwd#$MidCxb9u!eO+x zDW0i3cTP@P`Gr9FwbKajyZlI0x}I|}?V7>|w}iio5Cg{I`MKcew3D2++F8|?S%3aI)?xx|UoZ^a$xDYFP#+jF-rd_PLSMq%i!el_8q>2#*(bQ!{zCG1C5fRCL0@XofyjU?;|2uTY@`V{FyY=2n#2f7It}dk`)%7 zB|KP3(0?5sXuYM zv%8(s#DW)_pY40mqK@R;PNtK7sP53@jt6g)pTx;sn(VVr`$^`ze@M?;Cug2)pB=lI z@IBRpbvfOJoq7(>u)Df$+n)F8Ht+eUZq1%uwZ-4w^HFVaC%tvpbD*|3eNS?~d)$Mw z(kK8=kK-OJPusJCx7FhwyeVzZTHZE|d+=vzl=Ic#t>YfNI*ro48tjjI@ba`hyLj6> z?oUU0Eb7{R+=HcQdk*k%?xOsVN&BZG=Pl}SvWf29ZQO%{()Ki&@a;zm7L|lL@ZK

joVk3w{F+CUVkW9w{={v-<-WJHm+CnoOS!h^{VQzjuTJyX{e~e z71et~RXR6ZThnFTo3%BW>o(NwT8G2TS%oXp>-Mc%S$Am4n%#P-tvNLNt-6obRo8vM z|Mlytm8K@W?$Ej#(lqSWi_%nMxm9*n!k?_;{g>l-w~h#>@2!}Vf&OdI{&4yi?5LyP zswv+#Zp{;Ex9R>M(8}Ue*;F&5q{K@zld7&1X6E-s5zhVd&5Zn{UD~+d`n52Ar&%H! zz0)jKz~xzf`-rB+9{>JQW+iIOJg1@`$hrm6>r~v9#p^n;cBaA`mj4}G%Hi|43*X81ycU!3oxO|4}}u+`V{RkA*S7Ki0I&CVyOUZgTrkH z7lqxyqvPFV&%gm)Z(0#6j9zR1Kla`QKC0?!_?|!lg9IihXi(5d2T2e#QB+byGmyX; zoggaJU=c7NV#R_m0WSd(l31q0D7Cir)>f@=y*zEb)C-_u5@r&>3l|l+hy+kL!+<1Q z6K?tbYo9Y`CJAcW=lR~geZP+%nVGZpK5O6BUVE*z*Irv*KaV!cR#df=$*eP*eSC}6pp8DoijN$6k~x|GRdH5iR*|-( z_P$>7ANvww94i8v4zh*=_gty_y`TNKx+Ek@pM}sA%y1zX_1X2o0k~%GFN;p<~_T4TJw~g{#s9;SrSGbMSFY~ zgNS=X(DO|FgQc0Xw@3{kWD2E?#LJQhh;KiLfS5`NQTgyEn(rH9&%AW+Myldw0m1s^ ztH^}vJifqSzBsuzh%kF)xEF_`jG|}xnERR={X8(t zp_SFuyE71O4{3pl6!L1mytP^&kM%fjrM4uEQZxDEsS;?Vs2E&UjSC#N`ZcfjOv3%2 zrih?Cw@w|X>!}5b>L3@+)7>r{2DQ_l58!%{O})KvL{H$PwmMEH;p&l?8HHndDb40? zQ8J3LLY=M?AuVvHuIzGh=CqamJ{T3Ch2pMR`n)D8yoU*+woT;V4vKds(h8cq#ITls zZ8UkP22K8Q<6nC%`zYjfN@BY^hB3tSI??w%87s7^<3holw5DyS^InXwi6`kIA-fX& zZrXu#7ilm4et}+3=LZIS$&i1El#%$QxaO-944P8N(VaQmoiekRlKJ4t^jQYnS)8bq z4l5e7P<9`(B@`tqVsgCQi7SEWa)eHf5y-Lgyphs(Lg2VzyD39m2MmCdH*`VnmudO9 zzCyC;qN*3+jVUxaG9{8PyDjnj+m~Oj+_IIz-bnF{nlGmwg&0E5{BI!1s*L1E>DwI`lSGHiOxL(| z(l-!)+7zkBo&sgvap_ zwNS^Jru-9Q^7Uv>;4XXZNHu)XU?(2mUD&61cvj{4Xl|K5rM?cG5V1kmefFu{ak`9gfj_10FBHj;!~-&;8e>Wv$O%>G*v$ z#Pm6{XMIsPd*$5jo+0<7V$WbXJ4z?|HF*JV!Qba+@Aang+chLlJP$&Vu5t4L-Q)w| ze)_L@NA%E+5L6EN-ag5V4`SEwZP$?r700xrOzxmI7aJ)T;WzXa)LTj|qUr5kGl#KQ)RY!4vo!0NW?UsDD4 zM$kQv>PsL~R8yF5QJrF?aW(`9kpp_yqM@@{QR}?`ZKef3(8{tgVa5h zcnTN`R_Hg-?(ZIdHMkge8C+|twMmHtae@GBx~!bm#X?z#vwlWUSv!5J@xtRD!Yv^Df)Y)VmxVneQ0n;{bA(ujrc-3 zX1e4)5SCD-3nqSAxc+2%aZT0{&f}e$pLmdu+=FTAGe><+l;AsTXR@qv@k99qs%go` z*&p!uGGT)Z&dQ=g;f|^?825+_G@ak^O>Cr2;ZXotm0EPc0*R%>&LyX9>vMg_%62$l zhV|S5kX~Gc5VUt9YHn&7M6f>p)cM)V3brYBwfQ+dKxceSf_ga>ls#`<}q-GPTo z_=(ar8k1asfTN4lh>2vJA?_&ofxkU#C4ldK!TRt*Kqx+6-WFASUYv#cBC!Ib^KN%B zx@_$UMzP}tR>_5LBZRHd!@79ZTcZsBWRRQnC)v{x7Wg`IUis&m{*#`-d^Y9hq|2&0 z6>}Y;(amY5_v7bRw^&w53k=u9A0SD8@EReo%aui%mHJj*_dX=$n+#q&3vVHhVP6t{ ztMO#6r%$w5PcJ4hvdOalimoi19v7kun*vHFa>!~Oz?w=8w(5-xRfHR7E|;2D#FdLb zU2MJ8q6z}`dUlEDr|6ZNjyEH+dgM68IxJT1KptyDxw_3(6d!dd6^ec2{Mh`~{Ki$A z%{5%Gh53DrQV zkEceO|J`Q=4!%s5P<;>Q$cez@=S)YivR5PK zuOqY%V*WZ7EsgkVD|y*bKA%rS`rq(H{Ch}5Ss-)eTF7UdB><&2Uz6{Q6_JUG-#H!* z@eCLG8^@u|;6zKzoF#mRWjTqHFQ~!$Kxikrs6Oh)$2ifI_@3Y`0UCkO-c%(J;^F$h z)qjp=*SK@Mk_^0aF7IH(kq(oph)Rl*%$Ck({n~3qB{d^Dy)i_l5?w2uCGWz0mP88@ zn6#Od;bDOpvl7}MnoisB z_}a0!S~Ss}gcaKsmUB8W`X?{AZvi*EZJ4u^z5u%gL0*Vq^{?8jC0_(^q8Azqp7AD2gn4ZKkp#;u6flGe@6tB{hQL zU?(}UDOq-$-Zpm~yv`i@J4hLl%*!e=nTOhA31zW7o=|kI05c6WyVZSe$mmjUIM-%< z?NvsPQiM{p89GDleowJjHMIJFi50o(RNN*BM{^t8&A0|+DpRPIye((_;BFH-c7cJ?UE=qImVbtco^_K-3s5ka0i9&CIjEW zn+$vhfk&nxeeAo7O%2BAGR8=sJBStshJFA{zmRf_5^I6NYAIv}J4Soi6;W-Vw1KTk z%a4}!HO0vNL1t+cQrgJKB%#Q=Baq1o{KF)QYHH**f&C=X-wYp$F`5zZ=IW18pU6;2 zWFoJKTrMA%$Ol3zyi4-24yK8gl%(HDW^rsxia@kT9Y^(9)Lv>iTIV_g3dDbwI;i+2 zEXL;2eu>bfQranWo1FRR1ti`@HEumu@lDv2y4DT*zrlNBM7=exf(G z-h@*QDy}6;YH|#(!~$irJX8|g*@w}bzI7*5qB(tQ>{$wh3SCN|M@(9FFKRj-3Ux^= ze~|a*`s30(-yU&G@%WrhiFY9$m0AKC?|$ro43|EU+L#RPuqPWVj`B~WFJ#L=*t$J( z=3<$=tSKV%ILg)1X&4$2`n%=_rm(f}5fO@fsAqmuD!e^f#ZwZa5*j$k;5& zKLoJs6POU`Zh?7$g|Fp7D$ju~NBO4|DG~c3z2oI`Bz)CA6~ggNX}#4xM?Pn)P+Ozz z>!6xZ|MzkYuh)qCjJ^G!E?xbtJ6BxX8?CyZsRu`WokxbWfJh zO|2fx6j9_zQCH26`xa8fB#1MjEM7gqZ_?Ka{TZ1dOrAsPyH+n=+tMDQ>}RHXFe{mu zJUoHvEUA}z{MUH=Q@}V`wr6yA*@>2+-PS9Nn4*#_%J9kxnzHZkbe#RIRkCM`RdUui{g%swq1B2>7(MR<0_j8P z{AD8Ry@H?{6rCG6}Hw2l&g?rRNFltk$S{H9IIi)T-Q>T^D@vW0VK?46m#+Qeq4K9M4`6e+Fx-z;= z;_Ab}F@~?bdi146TyD!H2xrXk2wKWvRUM`K9)}>!*K3_1^$lurH*?@l3}%KURa|GJ z#SaWn1TOkN`>8g4tPnMZ_dA$m!aA0!?^@mU5%K9Peqrzmt?X2SqkOsQROnMA`>P!l z4Lpwu3{a**u@Ap!&^Uz|fhFIeL{4mQC-U$VvSPMon^On|H%VM0E92sesXibMZ6dvc zhZ@w#>Gie3OYh}$31_5%RC_udP>xV%X03N0JtCRuzoK9yWy<2L^{yD?4Z_@LoGHu7h2?xHrCsy3}^Rxw4EDWr_R@Jz#m@yrD zKJd^)b|WUrVp52DIT|ZYfoa8Y(30pfk19o4psa*qjb3?UnyoG zP}6?%hc@<`dq|A4-za(Q=F#FOENc<_5Ug+yO;rXf{ev;mvn&;U-0d>Bc$kyNM4|AW z&u2SFg-@zRW|x0<8h1QviTe4sft-01h3Wfn$7JDX54bTQnni{x{> zJ*z;!m2zCZ={R=cPtiPlqG+vZ(lBL|fA_qRQR`Z9U*%Td!`_X%*OEKf^M41iO@$dP zuyWmEVirS+1|aN}`{^ zAbX&09NlmY(0s|x$bi|?al*u` zY#6J)g&Y$40>h6S_7?40@5ubOw(=kBFO%ou`~)0KXW^7;L`ye$k{Lz~73RckNIT;} z7L(;Luv_XO9;^@|K_tBaY7;imS~4 z+F_CK#7iS{9)g{Wt4LJ?n7*R|lMk~u5=>=`MfFZGyh48^lz_iD#6myiA^{Uf2AwZN zGl>2f8xd5QiJ}3g%Zi@HI2_fY;}CvYHd7cRvCE{0wIV;`knCmA@NW<{+sz#8;)*yd zWaj0^>ibI!LR^NaGvKWl#;Hfu)mKhRegL$17<>(m5Ej#zhKEmA^`UQItJ?;0K0LS+|0wPcVF^`W?N6v-!Fxse8}Apy8~72lpXzwTCA&$q zk~h7dAnxAAVEzno=xbp{X>xl=u8mM}m!oYYYQEd-?&hF2iO4b^IRYKjt6fJp-X3(J ztE`6G6(U8S6ZK2VfSqukMwFcx7kbb{k1vAX4LKO=H>bVE zu?wuq(IXWX&}F(b`!2dm8M5^nr|Pae-=IG-V9MmI`b&8t?a@2J3EY8Rf0UYw4OR&s zGkHg~+~fPPGzX)~s~+Fa6swSa5wl=g_#8$HB`+TH{69zD02DVijz=fb0ji@}r{j*6 z;vhZrfo?1$J^BbQBFv_;JV9S;ps)HL36S%*{bK%MtS_?JgYi;)TRxE-a=w8lj7uNz z7_Z!^^JH-sa1ZA88e^l}_jwsOl~1Qj2{PKG8V+VF76=bz_kYso68Dm$9_vy(?I#m+ z7=v%RXn4?%ljC=W!rfb@AVT{tCg2@ZQwEd;O@h;USOmULwuPBGdqiza8Bx%2=6E%xKD0X zn)0KCQl$V&!ECXDA8eIu(-Vc&z|J{}BuaFJRHmgLkg_7iWhqocv`xQ`Ss%wg{0N@( z8+-d1)e?ElvFHy>)X=ecE?o=Ur&AGkz5m7&tci`9|3<>85+@K1QI193s1vN4d3Ix? zgwG}-S5;H{7AKn5p}u4}7L^Kn30$YbxzRaiuxg(YEo6Z52Ux{ve=E7=U0JB%o~HsUIFC#y%|vGp>7ovyV@IH_?P(ONIrJm+;j zKP~#(gMT6J>Dp0&UK{iLXXXt#YyOjwr?OTZsZJ>3-1AJrynTBuI}(=>=IxklLSHDM z6FI6TAsYv|Irv=9s*}5xCZ=?!t=+n>#jSVVouQ_A)uIc?zZn0e^7;N7d4bA{6|bAT zuo87uRJYle(1$R(diOe=_^dgE{#~m=@U4~luAWoH=enZK4|Z~8cboqvvH>nz$?j0o zhQyVOy=Nt3FJT$Moy?C$z6nvoXq8pzW;~OaJj?mOR0#=(+@Rc8qXL;{$XzMk4AxF` zcxXuKn5VYHMY%p7pBvfbPel~Bhm~iOY2Hvm505|MuN0-0$mXzv!(!}*@T)tI~S8}xZrnDwI*6IUpC zh~-w~p>5^`3B6zus=l%;AUWuMaJhZwOZ}03?xldl6|&ayFv)!92s8J`g-O{!t8+< zQ)tNsZ5?6e5@+MSvqBB;3S)Sy)$pG2pgr3$|7|fv3pN>DT+6$8RxrAX(X2?8Rf)!T zX9~A+ustss?)pxz)9WuX*h))}8oRDJPj#%iR;Cm7aEx8OSb@YCLSFpWJ7Dc-6y^Q= zuox92>F8lhBe4|Zb`pr_V}vRgl_^Ycidd{7Da=>X-i5vVSP@%ln~Hko51Eou#POxs zG90pcXP+;YF7ITb49H?MR%w&WVxvMu?`3S)}Rdy)AzrY&^Q z*xPGjZ?9vISxh2jEgaWG?>x#9YxN?ApWQ9mou2g*e?gPuwQPG-fd~Wya-!n z)P5Ofuk72>UU_TGuo=r#w8i~BR?JB_P5P>}PHm4h{;(DE@8`%=*`}Yzl+k}nOT_B- z2S`_dN7Og>GrSXT+!g-$FWED{pl8tzn97DdbGMY;%4+zp*)u1NkRJHoZO`1DD%925 zo|$#FbY87KmT$|Rd9}g8^<34%+w^DDcdhPx+puTG@iR^mX#NSbXCom1w58Ft8rMok z`7Nx~;uI=oa`vyjKOg|%X5s<@U;|_E{?<4~Alt3^88xOU1b5nf%D7=I~|J@O5k1*B5L&!ioo#b^cgPpB1oWL22J<+IL}Pccz%Iet^cTptjY254$ZAK z>!wUua5IitTWjoJ;(#Kj&|s#bk(7N^m|b7r87BmKNWZ z_=tRZ9w8EG@oi%sNt3T}k7Nd)EAA%uw-h62rpcGOzoobv_qWI;E%~_UV(F~*X5jm4 z>DTB39Ye=%s|+0{fM!`O_eIv=Ou98d#BK~aXcuJ?x z!SO2@56&=)*Ly=zL|fWq3hu%T%3h?VvL}a!xF;|R_4a!HZcaBfQBU@sS>rejkfvpP z3_Fr>y0yFFznnsojA^Yxi#{onMZC=s44FNaIF}g1_zMubQv;w9~UV> zN)ReE*5PT|QhxBS{#wnanRfrE-?PlFo(YD(O?5Zf4~~te#6M9(H3HSw?Q5pxgg}{z z5vLGZMJX0jw79Ez`vj{V>ed0{5k=3$ri=x*RwA4^Dt1zCxPwximF*@C$1!b5lyefJ z!cl$lC;!kda3@Qt)k zcR<%v6QF7N#R#$0CfT;K|I10can3aJI@GVkk?dNSfRUg5Q;Z^8)b*tLOJ_#G%ATY7 z`eVnjV-@~}nieu$XmqrR<>^Av9~i^m?1=PlrLRiYz<@z7IIOo0P+gJ=I_778Q-p8k zW%(J0;yVWQZ#!A!Jb_S=SWiwyjara>y3DogCNX7wD0hN6hHazvMt?ZUD|i#pZYy6l z{W!yF^-1u#hq`gVkiJvhpv+l2^ubJOBg?43G>#uqSy3b~5W){)uZvaB>$8m8hK7*r?V{nnkk{bmvPe8^Bg8W5&1>J@8xjsajG_CSQ=J2 z@jO4WDJkoc1lyLYo%*J($C|gWpS z(`8e$&9<06#uZTYG>gVjUG8r!Y}jq*5cYAo2#L=z&a~i;@KRwB1^^kC-+07r#d^81 z%iRMLY+J`U)71;Wzmo(E zs5mn395iAOB-YRV7OlMAt{vT=C2WXn*D_>_o|zTQF;4bSXi^HLgc@#LrSGTTBEO0m zS47|R|3Zbd8Y&U491H6CgvE|MC#zDvk^%#8@8?ICMdpv*Pou5~r0)VCaz17Hv|2l# z-r96Ni)*#QS)Zr3uWrx7nA)j@vp!C5|8e`mS)ugy8~ECn-hNyA&`-8(59MzOhw`>G zgznwaR5yg#Nn!1ivnxevH?Hg*ToYFntBbDhAZ@;jNXk z6~(nVAvx=UJ6C0jHna{xa8$@ynI+p^vp*?^``0D}cQ&0}lc1H#LH82P|2sQ-0-FEP z6uw-(wXV`Svd?#S5_6U3@?T6PmY8ot*{+16UzP3>B%UnHeDB!grP3XO!uLt(E_p@k zBqk8;@^@y)E331lv64kL`qEsfo}sRcI?GFN_QCVt4p&k|K0t( zA6KB3wSfYqdP}9s!5t5bO?s-o4EY0NyF6+B=<(8VBdPbJKw9y;(6Q;$*3|PkXiaeE zjgGJFp>+)C!L4@2F0@GoYSaD~)~sa-U^MLgR`YnA@C==%p9meBU%W;S?sPl8v4@zx z>Lp!X`rLS_PQCPr@lr&+w3nBrtzl8CN+dqUF0Hg_rjTvLY}k)hTcz6nqerEYn*SW% zhG^$~Vszd_XhL<}h1Ev4Nk%amX?RsUs-}TI5v>T;5$;R~{;VqB%D#Us_;CV`j~zKM z;HxU#zKEa*a25xNGFPkgGf80iU2$6d9-lD%L0K^T_p?OFI>o(rhTqgdyOIG^5Cu;s z!=>DCoDvrh?rZ6+9H;R9aB-*^C$<#PVK6VTb%C2xkhXgKX=*!$Z9EW9B?Df{s1QZx zMT(;G5tURy4 zlAoQFTAE6S1nx)hxSwzq*u;L$vMm>-vwPSFnF8ld`UlxZ&S5_X_r$T1!4$Vg2MVj#9)n z{cLKe|0-EkQ-89H0c=p;_&dEAw;vaCt@&a9uF(wff1rQYlO2=}BK}>=zrx-90h%Gk zky6z^+w@K9yH>yB&;P1_SFJG@p&G*;@&zUS&)7r$J^o!^Kt^rtAr~e|zt!q)zAbym z0AnubgH)$((-*1lTD|-KYGV&MjeYl$L^TxuUVC^3t^N1f!*~Db^!8nRf{7z_vc(p{ z@ph&z-ojtDUWC%9zbp?&qyDn+=s15_6&%lx|JZbPSS)|pj%xGrH2$)SguS7#p3YzP zAXFd=iR?y1{beV#@^fwBFFQ=>|7txk{Besi1vmQ%_{)0lh~jANEBlGsP!RuJ?^=&? z{|Q|2p>iifZk7v&S-QrFqOncSp(^HjAp23cydv9F={=8g7&hMJHQjPr#F-6(p8C9W zzZhwFKW;l4v)BfMVly-<~rUgC_M|tCv0pe`~3)pN77kw*8+$U%#dk{$2ES&yU2Z@gJqH zP3wPvzUn^n(Y}21K z*M$rJ*qXlB?{4GoKjc&swZ#Lq1^?pEV~l_CVDT^bR2={MfEnr${`KGY_rIK`{QLa< zf5R;M-{SAz>qsl&Y_We0^SkHw@moUi{GoL35|kq?V4mJZ{V0(0+7l72*#=NoL*zQA z0=XZx@mluX%bg`MKFVXqpY+&ya^(OT##A`827~rEHC#OLBH-Y+C-9~^qMNlP;%(ut z;qe*AjwSY*E*w+Se2Wra1Fse^jwtJ^pdiin7j-(B_)aR-#l`bi{>dA!QCofF z3GA&a%8s)Q@~xws8*OaMWPbyW%cTR4or^DXpv%A}(Xlt;lDz3ng0w9;hYXk|$sHt;^L5pP`;+NM`>T9!AasUcrQmW%&zf@A7;sBM-sl^WmQ~dbjO$cUkn5sp| z=L%+E91y|g7U>@6|H6JvJ9c}zu@z-hfEu@~fOL{7hXrey)Jq?%_yeU#Y^NlB)k*pr z^@v)C_o8yre@nsMGmcq9A*I*wk@KVLv)Is9QUnVyKbM~YVxr4CmakQVEwzw^ zSFcb_piA8Pkz9LPQqlR+j|H;loW2y~MmFK`>H1hs;7I$VxsLLOIFGDvfMCM+@#`NT z#~+jTpCiX5P12#9I>!cEbgu4(p354OFgaFK26xOlTKR4w6?Ein^aNfoq7a~FPo-bv ze1bFr!|Fuygu=aaaCF};*IB31IcLbIokg|K5le$IiQ@#GR`wU^PaUc<;`w+vpFm`% z*xgih`iCSEpEoe{cM_@UW2&xil-~K2Iuh3#iAXwD?|wO*hy=1otrW`3Fh(!=dIeZC ze_MLd^m`;lp;*TbgrIQ8P{@$r6BbW=MSnjHo}wiNibqTo9YCRXQ)R}}Z74iT3_aom z*|sY0hb69^O0-D-6S`M}m}P?K&mE1{w4Yhb9~rq;{fYYp8v~I18@$3k6O3c)WmuSx z2npu`TG&04TmK}4jqFRU)3U1_>i#iyi$^-%?rwLL92-}($sM>cfuvb$^Bl{QQ}O~g zzThf5mO7`~s7cAcawn{EHE(y-Bxa6_yyGfs>f^nZ&wfQ61=KI{f=U4$*C+7mLQiG_ zr9}b)J|pt5e02#I(^UD@DLm8s)knEm3$F|7N17~oI8I(H-CK|(BlM}}!zJmYaD$18 znaiqBKy!^K9@1awL=w=mL=J^FJVA;^C<8gSGX?f6&jSZ){94Q}sMA(c?v&F{HW!-~ zDm18L(b8VL6ewHFT$M8J=l2ZtyWF{vS9lz4KCV@doMa^14B*1?rMQ(wHr;%zG#*!) zkH*qgR!^ytSvOMGnvZqH<4fkFShc~NqTbDANeYx@ns^#X!rEDoG#mqHsA+z=GljNm z{-m3`v#f}v7woT*$)%3)$dQrQGC7)#yP7hn!T@KayYgrA4{*AgYx0PQ8u;Kmj98Ou z@R1GS_Z$(|_5#_@+jv-3fo5fEa1j8>w_QS6oTV=VC``(@k)93p`iA-AwQt3w*bLkJTHEpJ;(^6Yx$GJlg_aC*Y4v@XHqX zS^>Xhf=ey%#RC3=3I3S{?jzt|nBZSo;By2#!vsHLfx8QMtO@p7;C2EYW`b|Gz(+0x ze6b0hYJnRB+}#8hTHsv*KDpm${5T8zg@AXN;9D&4N&&Ai!FO2TnG&C!{75Oz@>vZZ!cl6P#*+cS!Cm6I@_{HwyS%6MT~e{#d~6O>nFxYF_T$ zXEZ+6-)dfdZi2hA!)Gw+C4ywR366Eu3j%)81W&Vyd`iHJO>m#oD3XT-{D2AWY=LJB zxX=Xu<;-aA=>opq1pnCrPZjVW6a1_NzFok*P4L?m_$C2&Fu@;LU`3ky_o~KM3_;mN z=w(yr_L-fxh6-gAm1WGIK{v6zb9req_3j((&(aV3dKj*4`! zAXq0ZMl@QERLM@7I&UU7)!ncE{Zf&k3{3hh6dZQ22riDwI@uH$m36ehUz83nT9p)o zwbW650~tj4*}|*J6pI2$C?>VlmJvlP%3mrb*DG1{M9?xS$wJz2$+yLuE#7gI?>&S%K9Z`3cknCP zyPZL9Q44Lt2>((l?UfE)MkO_Wu~Ta$A0L4Gg`WdStV<@)_dnbv?(p|?$(OJdvrEG8 z3JxR^*Va+7j^ENf^b33K^5EObWnC!!_M${-vu`-3XU)DrPG3Uz%Aq{F{!{AYAD$?u zBAJMf)^jVA-j70fA-t0Z@kA=yQNp>UiCN8|lhc!aOniL4yWGV0l*NS&bJM+^Yfh@~ zG1r_@-{Y=nR^QHR3LEBU%vZ_t`}0lm{EPV}dH#U;$=7IQ`%5%Vjx1H-&eu}hOEVnh zvZar^L{f^B#PuSneA9_q+3tD{WXW|uF_$39v3rxdfv?m2DI1<9Y=Z$E|)o|?7^SR3*m(%e%HHMCg3B34S%uds5 z`b(;zF#7#MnI**u&1YzVXVi=j$Cg@MCdB zyu*V2okirGzKB0uXca*yd8>$`=fZ>IFyH=tm?OJRTlw4==KJF?J1T~=u!J|)$GWB& zA^tn-9XLJqU1?UmIk0Ed+fnfi-cI4-IPA}TANEV5*sVSvsHzRIaY8L8vjys33hE2v zP>=XN)SrE0c8i62y{fZ`dQc10mkR2d{jolYpi6ycpFGe8^_+OrgIl2f3?pdx9#EU4 ziLo3P-P_{Injcq|R94np_^?5mT_}qOj`^GwUq*4bk+n2B35UjEh^^#8*`It(FPK6D zIM!Ss!)A0r$wH~f>js)Q$)z5JHr8}wXpOp*wnVld9$ASXo7@K3pS}~>pq9w4h({(n z$>DR_AiL!|k?r0Y-%Mkz9e#e4L`xS7zFIn-4vTAPhwsGqyB7HLClo5P+ZGA32iqW9 ziZFKiZo94}vg_lKJub)wwm~-SJCU8-5#Lh%RPVR}eL|4QEtxHwdBJxgd#wdBquY`d zMqzrT&j>ydMqA=rgSqGQEuGi`pMFPt*d6tAVKU)xL{09v(YJqR$xm*N?_;A|4r59**8*J0qO&329?DJ$leIqYF zO|g%jYxQ!k@Y!Nmk1P4swk3p~gjM(~LkAe@-#8&9*jtqFv>>`q5X}Hl{CmH$-h0q|@3y!K zd@S!}%X@=k6}VVc;6f@uX}RWGx5vHJ{sP`=FK?Y7ObMATFf)yd%XGbD`buc!3zdnP zT2nGWp{@Zy9pjioM1llQii(TT=`Jg3@mA@!_srdqH`QvbAH@<8o)SYV96cdCJqGBf z0OMl-y8`65DR}007TjJ6ww7{(D@_oK!4ro1=POMCXSfb$E&X>-oNBhT_hQ-AF)R8v zDk_R;NppA|8Cy2`2)k3UMoZWbq~jd@mtV#THgs8#a6VA@GioBaa@pEed)M2(lC1KG zio)SI;(Cx(wzbkN+l8RW(XUY57~5ZVR6Nbv5N)62IY;j%9Ks);XHKg;se11%t@GSW z9=WdWoU+D&-cDtW7dt9))N51~5$L0{&21(d@`YR!evUMP9+Fzkr)4=MCkHwnCvZjj zBB$gD)!%CDNrjF-wyLmUcBd2xU167y&8=F$J#K__Cl3T97A2Hv;;5L)?G0g$${3uy zrsN_9%NF)X&}f4l;q3v<42c1p3eX=w{QS9*z3o=~cP$z9LEE70_7oDI+1WPpM#8Yn)u8|Mc|5GKZ$#-Z)HToe<7!F{ps?}(%p;HhMf38Nbv0|)-?(P=&^?{{r8Mn zcqYYSVj_S9Aw&5sVJyb*rhWsdui@)&?BGdM>NxswR4jvwh29Daz7rfolt zzB0fWK2JdL8Sf-xRnI4`d zKc|N86tH*)hPl(n7CuQ)k(q#->Dvmbc`g0+wl~KMo4oB#XxzEimlSVujS)Q`S?OU*7FgUUo}U3(ZcV^$D5}~ z{hcKIK|X%lCbej;I>{fI-x6ifl*o)$kTH?*k+A}cQt6$a;ze~`BoA2RXOU|nS4X(} zJ0#b1+q_*PDUkuuC&M~?It?jV&wVnJQ8{A@+Qmq1y>iv2+Bih32;iDbe| zd5hHLXej3N)gW^t;m?vV*h;w6N~n>9Oe^7RE8#0i7-c1#q+$lD?hrssxYxh767nQr zg7wCSR>C4l=xZgsVI@2%36=t(KWQa=Aqk_cH%hDo2lIFNcq`#9E8!=Spjio{RKhg` z$)wK~7?b{()lEUkjNWf{(}fn6Rgz#SF1p=HI3@`C?13n( zJFvG6*qsK9=orH@B|HWx?O-#nCFs{jbK_!m#Ry>V?Wg>nX>k}hZO)$H+SEb7dagvnMynM%0E|LbP}QNLf3OjitumGVNYlxh;G zpCgv|Y%KARk$3=>XH4bGQC*i`MFwNgdsB|3)Typo?mx zBn}^Brff4(4z5?@Eh&S|lyuD%y^d^hwr>1FRQcl!R-ogtM%KPb48W5sz7;b1ov$a!iWmyxDGa zHiNJh7@gaLbUNI(R_bOf(i{yV7}EG~k2T`Ln`*Z)d5>O>lo`9N+E*Ri zaq!40iDSe?mfQiB>?8a`S!1U;eb7O((*79=uTd+xM- z4f}?1AbG@Uq0;VLYI-=*ggA#tclM!KT|60_Xy2yah+9L%F}n+4h29~ovX@9$T(sWyTzX(=!y(rC z$5u(zut}=WP)YjC4@DL|s^12w#{36F9_7>+?bWkJpMtJOB!f1QyK=hwh9^sq&=D-) z-55>dAbfXsKrSK~o`&1Y!>J6?!r?jcv#0v$a``*>+RL--TP_?vOn!>VdExLp`6=e; zg~Lb4Pcd;W96nrriYa{I@R7vZNf|MSNGwE3ineO}3T0l2tFN3F9dk36Wy%U|)LaY5 zwOuoBh*m$>IY!k#*E!B=0Q$1gf?VfhqX}|#9)+tm7_Ra1sM=BP{1wQhEgYQ^sA^5Q z^D%+)s9IF+{EhjjT2=1+t@)^0R_^?@`KVe~?i5AGY9S6nT6v1f%kBvlUMJ4U%l$(V!)ugY`JKt zHgA8o|84Qjcl&=^&lf>#ydC*1-uo{9C_=k8>#l@m+}Mlm_V|7zw=&3>lm?~O#mb<< zmHCst`C6F)-2O*YV=x)8E^-6y9elwm$3t_%TGb!EC1+ z8B&c5Jrwco*5_885?6e^=)F_w9FEMba3R`!^t@6mz5@43%_ofi9CZu3rebd)-B#^r zk5%{2HL+tnoYZP@3;Q6md12Sersl;mV+QlXyia6FoBKr0BQdfm+9l?BJ_wX$KwT>0 z@n5UY2?{@6k?T(rV>-Q!^%$npOc;-3#VOc7NEKb`KdbCmA20E?k9fIz zrz7OXjWkK~CptYDjovhWXF2iG*Q`p1id9AJhzn{gd+K(Qz|cm7M7=De z>akaT&oS#EIjM$D8>8bRywyL)t__)!!e!-h-;%GZhrscHVO*_N>Fws>x*TlyzELTW zuHkLyyb?N*vlHDgpI;|%{@Q7^!|c+JHN!a2S2og~w!(_X7ae~U>G8N{5{H@QEB@Yz zsK$im_I4NY_{ZAY#HfE~d)wZ}@|2of++Xx7 ze#D=rmH2o*qO5SYA*0PjjM!SDfuZ|W$>JDNw;jfZ^pVq0>OwSKMd9H_0=Y1Y2ZHDr zVSNbjK!liRSf6DwSpL*}i7@WJo(cQ4_W?I?Na5!82zQLz%-A`ag!2iInQK{!F7OPQ z#C2#Y5+ep9$a9C@oyrS)e@P)cPh#EOAs89)XtW{X`Wmr;K<0fXV#Ifd;V;S(+#R*D zhY0U5S@SC#y<6c}8;@g+Y$f5IFj(%)k$o`3&?#YkxSFL({D{-^ixoMpc!O;ikrv}) zIS{#re6{)$6Hmu~e>lF)4BFf#zRiD8k5=)23A1Fy|7~@D8AsFq#rwmFcc_-Ev4}=9h-fr}h(=dC z^Zk>BAox(RE&4*Ft-@QZ2K_%3ejr>)VT`36v5nC-;u;yF8M}&5g|*nW+-Si_W`|yF ztPT2V)x2$bAG2AnjE_ISZpH5~MX`VAezuNk_8~Q!;_^B4=>Jsw9vRpF-SLdip@-YX z@44fB>CIZ*$F~)~XPPncx>xn?HvJ>@U8|3|UFJ}{egeTj6~yk}AaU9e{+c<_pX@!i z>|}FM590NwxQC2Popl*{gUAU*XGBK0eEVdXzNW0HxhNZHmmDKf9cAK>!AZ|%N5w7t ziVT83vC21(7WFXM00y!S9&aCDbBX4PeS;$c|N8*W|Emi0i zt%9Sh8yTnt%cjSF5BI9vnXn93OLhtB(NX+2v#@_Q`@*<|?c1#NS*!FjrRAKoPLXQx z?y!Xe($zg*9H);7Eh83+cfdWOvJR`sO&627+48|!b0ybMqnMb0;dDisbih5*0aIcf z@EnzKm7Qqz-W49BxQo$#I>0PFiel#bR^PaMLGCds62;A^A6cb)UTCF{M0@K5omM?~ zt)u(^g-IB7$~WuPLFyHma5~gh4I(+l0eeQpf>FMsA{W_K>dqN`5Tiqan@Ys|`QIah z+*o7wzB_PltKQeLKX*L*7Q5taMqn}1&?=4h!;g~7s)saNRiP3lClum6m*V3I!BKIL zq_8*>8SErJ)`itvs|2AmlhDFzc`+uBy37k?y6l$T&^-G<$7N-bLA%lzRn}wM77s(eP9rQNzYcXAiAl0 zK_%3L^6NmK&$nOi^9&;3kc#K6A&ga4%tmY$ABCa_4cKh<{kumvV z_0<~rZq$4^^@OZwq+PZe8eA4i(*jzW9B%exle*H|Gv9YFZ)x>}GcI}`3lBjy?Pb zlue^R2)dq_-9_(Fpt>-H0^gMa<+h?E{mgfTnh+>FzBG@IAS7Hb?Me7X^G$7iVSs88 zWc*QN4VsG^kw^b#iD^HJ%Ckyx`=R?!8MXh|`vYdZBdfzUealkT+FNJ|@_;^GEg9SN zx6P&Fnp@-7+JE`}fZtJuvQnAr?Tb?OfAIYQFG4MC*V~4-q!VlPo^Qvkw>eb?$=|KxP*?Z4}Ok0T5SlAhw>laSL8*0^sd(&~PXdKyS9arA)iyWe9qxzI!8 z=EN5FdmNxAtouFgq`-gTevfUtnWHWc$Lm+6dzGJs?{Vj|aH`T{HaM|k(Rw%^vY<#T zn5ZPBN~cA}PXZCACarUDtmy>Ni=O2L1z8b=hy$I1ycLDiAzLWOi&01=kgP2PH`IKF zmtr_XAyNB+oV&Wu>vXi^h*ppNcrz3p+?U=r*V%PXVMCFmq$HBk`>;wmncnxqz2+O; zIsor`Qo#?W_YLkPK#uZw5uT<6s|Y3BA;rvlRyz~xnNh6W6AKT6_ob;KQUl}Vu`%f)6bfmH*GoR##DM|^$6jwi0L zco(h;HD+KlBqelgLKU`I420Z{MxJAYTQ6`2Zq13|5hQC(B$M?U1cSU?V!SPHMqi(- zULWla{H}!ZTVkj)F-+CFL@~^0i(#t5pn;*X4TgFX!!-RvO#iA56Wd~#rZD7!!A4@s zI@lO?>53V;FN$GwTMRQ4h8!^Dw80=-X@bG4kBnlFO!Q)WKX?^}tEC^>U=T@IFx;;v zMKNTy#c;pEFj)Gb4Tek;L$SVsZ4=cGX>BpE3C>Usl71k6%YMjF7#KqOSf3X~lF}N9 zkwM|fl-_8AXS9i@T<}~O#baxWM=~fp1Ef#d;F)ORStxkEg~nB*>eWke9czT6QFzj& zXWHPIVd9ZpdHv5(JaujH7~A&rk5l@G$eEa5rwyKGO*|ry>7S6S`scN_cqD_W5BEm=uCfiD*GxR( zE}%awc#cg-W^3S*F8P0i{XN`Xja^OHz3>IaB*-a0XBau<=S0aF+3V9ltZ2U4a)kibE;H=}PLRy&++B>$Azfai(fi;plw6}nVf)#}a=A~y< zs?YVI@U%*{cIGE`2pt$7iu8)fx6ys8v5TA~g6PCpSqQAIquWA< z3J9rB*R3)-Fw``Ss|@a|mw@bDP9b|2{`#=JZ{kIJf~EB5MWM~(xutK8?xYsi2gY4= zlPdfsu}Enn9q&xOXtdlJI-0TuGI^8KVM;3*;81IQNl9F2)@zHqnH!Q;7VTktHy8DLu&;?# zZt;@KqFlZjSJ^J^OWi_$_pL}u5O@4MOwj|BJt4HL|H!$th+p1Ij4y&W-C^>n?SKw_@r)suqD`&43R^|%8dI~A%fI1#LR zVBQH=Dd!;O+gznV!RN|8RNUU>J48+Le7hCWVYUM&Ksvc+LY^EGchS=$78dtR0Cji} z57IH=i`f+1-@14;Cw2ydE9!cusO7&p!YqA5boutz!(f*W65*if8$YmYUo zQ7&J#t87QdvS$0Nsphk_q5e0Zx9(u7K@Z=1f+=#FjZz6fmh}&h84>p=Hig1?<=|XD=xe+JM@> za#ZBg?fLA5UyeIRPcF=T3d@>U{4D#)aN2W?pbajdJ?}~@?LLXv-BzLSwMT5)y)3KML<0fI)D&?e7ajLWCT{LLOfkY!a&of0~;jQvFRS`)M7)jCkU z>|DlkZZ_LvGyCgG;siKtS)zgxr+sGsIwf)foxIGhpzT}bO;*svR(VqtG^thI?h4wW zRo+wu?bs@>k~_IE)~;nYIx6z$QZ^sF{k4Q9Es(e@YrQF-pmTJiV|J_7ES4t7#kbiT zicpVG3iMmohxXjQsjy*k>aATeaWKMy)Y)`Kyd4EK&fgh-f&ydb6vySLwf+?-SUflI zX;Nl-oc`E1NXZY}l@Q$VKx)!ftZiuLI0A9OVBy??9!|Ciwf7-)8*kb|^X3y`U#!_F zW_@MvB2~L*;~0zp#l10F%+Xi9CI$@*WXTG4lfFa%VpB;DJutoZxnK;|*TwsFO;a*YIXuqd&jP}G&WB#JC^8n&f4o%CI4k^*; zk=-@6nZ?ewnj(B_fjNmdkxx{_uI5R|XU>y}JQgIqEsss}apn%XGH=B^@wIpw=BSgh zGJG+8Qtc?csSK=Y%j)VQSmb2Os$AJ@jr}F=?!fY)Hm;2H*jCe<>c$PVcjZ#A8oN%V zY*%$nGm+Qxvaw@Tt~P_BNKJ@4nLv+x>hAH;yRS8(XJ}hkbd8d@MS~HOmYjRZR zL%A8Gtdif&RHzC+QeMWl8V&|Jw-}O|+*R8<1QW29INViriDJI;I1&rD7uzYZL*eVa z>|?46*PTxh$%XgL^QAQMF6Q9E=e+hYs|#ywg1_;an7&og9_+#Ujg;;xMXabQWj$8otwsv>hx1NaG^HB5 z@~1{hFO`xhDX&XGg?EgyPdH|zrKz->Xxfg#gz@%hSs0qZm=aA}U6?T29xW|hrLC13 z3eM`ngn4%Qx^$jBWt^k4^i;+;XXnBTC)+5mw5x4GlbM!mrk$bEb`(B$zHP#OGp&o6 z)=8zUFMRG2+k|h-w612FU8PkQK6j06!j~$|>u{(vud`#}a~>)0xly(Wp9<`f9=AMf zvyBRrCkn9@b#VBfM_DL*y@Mi-#$S14FfX!KuT3aiccs1hE51h9tH0sv7JKy`zM!J| z1SrT}eJVkSv#^0dU!Ba@`%=CvZ0KOC{)i0@@FYTvli3hxTr4;nrgW0zvu#3yl1xH6 z8@1A14O7lYVJW`IMyWu{)wTxE&M6xUe|4peN+_rr{DvvX0v%zi{@Q@5fp3`7MWDCX zsG`cN2EAcQSAmYVQDX&F1Ku#j0rd4LHfeyACPS@inW$>HQ2}L)6)-ans0LSRYvg&& zs65qFuRLo!Qa|;$*m$I3>aoOlq((fl9Sd6O!BkD^Lc_2X?iA@4@r)HdY1e*cwQ#`5 zEv9_HvZV7D3RVs4e?Y|F6`}QGzm*?*@r@VYY?j20p*>?a$dAuO zPIi|yF~8-EjtHN*kDLf?9=kHMY}_i5I#*j#=dt#-99wY**E0Ebl^#3Ue64LRk0yj-m$*wW*I!x6ZWW!m+w;0a&M&p3W=+Dm#w z?Il>mjoR)sYddIRX!Y1#!yLgw5;HW)H9|EsCv##bs1GFKHnAiFR5zwR&ROaqA81AX8l#g z=k72nZRK&4kE8n319R8dUDChbNFBC29u~ht>hy90CD+jTmDrU(WHdg)v|r_TxG%|~ zVz`>wElU52zt6(mfGy6eSnnwBM-U>jBYG)zA6xJK_5QeN!GCOjoWpfkKhz(xe=9`! z57Wo6(*MN%kUb4ZECZ&*t`96?MZ{zg;f`z_8uYXK=nwbTj(>JOomui9*k5WoZ6q=} z{gV_6*KKNW_utpCB*3`ODye-A@(Du&X%lf_NW0zi9Z<2lWJ&zC>X4>-?v z)#jc*m2tPZ`7u#`C@N~aWUUAQf0bv}AzxU&$5&%{xoq)rh^jfErgD4YFuUiXiB)?N zeO2qiCEHI0cieY$wXeEtM|%?bOhw<2p!41dEKZBcmAJXAU=_OAD#!9wWla~%J-e)F zd+~XwLe*(x>^SGoa=}!NZRrJ+O48hKmex^HEdAjUE*(c!PFHUgyw4itxWTIne^qHtZfkAUEgw-*h%fv1wD#Nuw+q(+y$`_&~?Cl(se zmM=?Qsfy%%sYPCaMt&owW!ts(8Hm$2vByO9%%SZuZhdOFO;$zhVrjWz85@NVgZv)4 zoZl_1E?(r$dGp0mbI&(FAC9S|-!vJ(@qcV;F5G_7K0PVO8(3Ay!6&(|_viLQUrlrl z=f_lxG(^mvp6m?nysyv7c7+G#+Kbm#>Fv~Gd%WNKF|H4c?d2p>-zPzZ`Dx^5Y`f)s z8QghyyHArt;VnIW1-$TJk-cw^-}0*weYd+4)ni_AKlvQf&n{o%`lZO%c3V?*c0kMc5Sfg!ddImg07#e-&r_wuKhth=<2;?SK<2E_KQ9)ys>viWVhH{z77t< z1uE$Jf7p8$_$aGuZ}+p&wlpW*R|K$d+oK?-p{xNZ0#)1(-!PV<5bOy_T?5vg0)9pTKNYcP zMKP|*?4@f``)G#bdgI8qKT49%ll4PqKkTa?@}}1h5qXk+s8Hiu+h7X7cfk4NL!s0U z69Oq>%B%4E!&rep! zd^TyR+O;?vwN$^>Li}2YPYbC+2p!;o)ijwJ9(fFZ#P=(fLGObRi}XvEyoSoM>9SA{ z;7bkR2g!zn$_owM}nY573Mrtz2g$rHKPpq__7rQWcMGhM?VKnZSdG^{x$NCCwXx^R0}|(*A~jjSiqyt< zYUn|elOL1G$Mrmg zrp-Cf@@7OoTd+5JI{P8!L)e+_$IBzE1C4%A`yz}4pU4&5*F5zl7Do2o+)*|G_pYFS zQOJY22e}Y3t0ml_9P)O;i=vz`evQ1fI;RYpI~3FjkthNQ2*xX`r#{QMbw!UaJ5Gw( zth`I=-c*ziAl|%Wn~;fA#a$WE*7(GsaNbS710rmAGM%jb`f6GDTFbW*SU8t?_VmpL@#&S7yXG2d`RL}}uP znT2g#kb5m1x8k@xq%iD!=oJO5V% z%Z64Vw0fkUfOrGG)cCrm1<`4Uyf>FxPy%aFB;1XJBwXfTb-8MO?JqV# z9NmNNd{FCKGAM?1GdKf%yB~e~NUj(on~OzH)A$JTnJ#*|cJ$REBN!tiBeFq8GQf*p z)JIZ9q=V;-G=?H&{sI=OT_A!jAc6>LT1=EMT~I>V=&?rvbc{0f^T8O^_y(N#{GKE~ z8nt<-5UOsS{D}Pqo_Vfb#xn+nzlydqN54hu6=?7))<$VmttfuYu>OlIMVZqv?rj{_ zgWc_)r;YFV_D}S?b>j#04CR3Arrco_9cAM)A#fwVC$Sq7;}4WQ-3cG0?IXF!s$nC( zscRcF2!G24VgEUbo;Fz8c7T#b=U6(Af)3sS9UQfQ5GhV4lh|?YGOSOJOuvTN9=l-*!4FwSOa9u(8qa& z;I9(==j$bWKpOZDC468SI4l)7e;Qcgb@eq7)8aN5f1ivT^oDOmj zJP8N7@;6ENkTmHZlJKEv;J=gbi_^d#k?>(@;3PIt{VquZe?r2Cr-A=P!VPKQPf2)o z8hEFKk4OWLNcg2`;M*kpvNZ6W58FS(i~Zy*wa)?L>f zNboPL5tRzq1wgc?WjgK0)3$tUEB4*jp6vwGukB@@ci58cNnk_Jk=ZF#24#xg-fHa!ee?wnUvvTbf*1G7bv|({iL`@$K_N-{ zf;h_oLb9kp3YD}z2?M$02u!tDG2x&_<7Lasv<$glshf2E}t zWkVC8mFed+{grv2mL^S^Rb1xAv}Jm=^nJ>FG;Nu;YiZJy>E|+cr!8}emR{nRuzE}@ zZ6Ec>?|mX`i2~sE8jK|JylvhCFfDjXi5Dt>FAzOTGI0=XSAdnWm2{0)vdDK2hp(e zC}*_PkIK}!Jr8QBcZ<~LQqE|pcgxhqo;$VFowDq6DQC3g$7J$xJy#<+DQE1S6V%md zbu&M8o=-OueSsI(|Nqp@Y<>R1bu&4LtLbJAZs<=pa{_`&PZXe3-Hhew93UB+f7sBA zulITX?^zYQe$daXfn5cB1I7N|WN26Fb527&gQbR4&lIyh2I?3J&&*+MOh?UEQO10m zDxIQ?Il5RVV|;y;F(0fyw=!lelrew)uP9?4$I<25iC1H{5gDaZ#)wF&h?GtlBO;BV zNSS|5ql}?AWfR99*^o{dqtm~v{ose_FEXiyvR`L3>c*x3Ut|S6N!(%-`m`)t$CArn zMooJH8yq}0LtTdne*W~$^m-Qid5e!4llLpRKgk$&ZIoEhIu)53)(_x)2dL<5o?{%0 zdquKHH;L<1G166b)K7pl^-WduW-NtXX9B6mOK@2k3?gxJ0)kde7|=7G0_eDVE)1~; z9r_+s0$YZQvFkVKEu2V}+D_76L+MlXSjOW6V25A^<-#D&-gXctZ;sGcWpwmP^2%QN zgX8Adm3!6R@z++=8?xfvYV?(5`!l1jbiWI@`R(Yby9->O+?sXGj_B)0Kc>0J_d)dZ z9XJya{vB3~t455aM2oZHsgS^%#YxYZ%mtks(q+ffX{C9y&z-J>)8+I{mlIF-CRg({ zRI{6^iG}N`5w~M6uM-TqbwKB#0iA~ju;R1hrV`qIw!&OP$i6uAWmrew4Ys%Kq8rAj zUva5j2)EmHj4LE@)dm_WS#dRew+O5|1I%4JBQsYB#$|E)0RjUwlHdzxWR|zR2HrRP z1zTXTjFamQLQKva+}SfA7c%leDBN^1@~d1L&a~cO%fN+#XDin2${=O0tqt{Dfy`4TH>PJaldN8M?@4(?(u+4=wR* z3bv!;Sz}sAll8)TNTq6122yNuH`N0p;QNN3Z0UL{vuwx?bTjxx)??v4P$W0qLx$R! zu%uB98zzH-DzIF%#

%A+McO_wDfPBF~-#jof?BdSJme6`Qc24O{Upq6&NQt4t!G zsUK}AS${Ml1HiGZ_lQC~wQ=}_=ay!CC&5KYN7O&5JfY+-U_!XxqiHD&c& z`x4c1?bB6f6RH!x*36cxT4Pgnbv;i_!s-l}0kSr>l8JLXdgjse|b1Xiu%U%*l1U zAZznBob|iQvDvlkh9TL{r9B$*EG!6M+lQV1=*oO#nS3;4WIsnkhPoabhrT^QGuR&hLokA~)>~h5J zs0N`EBCiUFI3co;Ac+$qUkaGYsTOOnbDaI{qoNE36Y7l zg!(4sw!L&tgs6=y7=MqgOFk#EC&Tp^&U_rm;1eS6;(-$)&*LlwPIX+$=R|ghmu%M_ z920>#0zro3z(dyNUE+mrKw-Mufs-QzZP_?Rk~lds6gC16@nI;OGZ~t_pO1~mBOFW? z$GsP4@8$y|N01+ma4G;W|;=~gl8+lE@h#DA{t-&gl&3gr` zRfD0W)wt;!t+m?3*N2QJnEhw1{e5s25}GL9l%FOjduz~O+R#+<`J z@rq{g>u&KnRK9kKSE>q~TiKaGHl$4kTDN6SG;j)Gx(dxV6!||Lirdv3NX7{{ziEn`7XR<-}>sE`=Yngx$5je)>{}G zr{tMJy7eexQE%%$IxnFwu<$qEaF5IG=~w-Ie~N5bU#=$wtR z|4uD92&#@mEY;V28reJqRQ=DeV1?%}x^GXk8V7c+TW~1i8uIW#>e-jbw?EN+#6?>7 z{pg$Mz8@_(1R8joXyEN{uJ;UD?+t@T^xVV&vgh_v&(T-UBVAYGh}7?K`Uo9|>3Rp; zYld}~5oq)3oahC1pw@5a?5?v<#=GmvJNM!=HFwu;yVhNNmN&t7Hk_{#{Fd5ZTeSW< z#Oy}>#q2gju-h)`ubuxl^j94;Q3+0)hd1UlEZDUE+K2vnJJ|S*_f_>@;(bM|B?$wT zNHkgK1jVCQU`Z6$r{XF*Y(PSVrk%H%)*9Ws;A1*JgO%|X*qGyv!Yv4jBF8SQSpKk+ z1gC=Mmg7#s+ML@a?U(~%`0E1PAyEAAr-+Bkx9-F+Kc9#BQ;AbFG@`8@BWH+J*XobR ztOT_myi8Ac)ZP;#D|LA9I z!Ue?fVUlkYU;lRV|9N~xzmIcJ^5|q7TSe!_7M(XrhpjIdTdf#W1-@E8{o447SO)Y@ zUe5~hYO44O=o+h$mr+k%EocedhRF3zrmqA2)7O7DzN&DtMlfc2`XciInzB;(@qGB2 zX8r#HUqAZh>5CchTIgedC|sWu{7L~I(bB3+mE zgCO&{8SqkOz%C3|!+5xgk5#?kJ`Bb@j~p3|EinD~MB-E@jxu1?kLz&wt_BLa>LxV% z6zpxW?Xlo{bi~f)sn1nOAAt*P*Iz>m6B7T*piZF6a}`Jv@W6temO$qa1)*hmKvO)) z^K0Um8vpI2+jX(HUbFt1PLx0w;~24__npN217mC4mm9UN{gM_^IDd^3DAEl2mmb=A z!$)u<7Ui}WcrtDby|f%l*8%?o`4tE6enEe0f1y(K;|K4~Rb@n+KWZaHLU{B??ly4H zxsKd@j2Jl9wSF|MosZD{q)<8by$4s{A4&UT2Y>_GA1kE&u`Eq#cos%5eCH5$K%~Yn z3l}>dUTvIz$(cVwoFLz*FErm($2DKxQTXT{l0w&^v&J>@Bmj9I;BRPh=!3a_l*(SL zz2Qaebn`zcBCS@>0ip!D>Ihp6e}3xf-FPEX`OXqLKX=yX6G_>d$PhihQRWR@;-l=P zvvvniEo`7oe^fjt$#+wyc+dOcTr{|2{nedFLA|yO0t&uP@ECfPuZ5xQw8?V3j#%!x zJ9Aj{`M_RWId#|FPYr9OLEXmv?1E zX00yn%z)-EEAQ=-5sM7(%!t(WUcL}#7dYS8$!N<<3v+WQ@_`KOKV`ejcH`OY^!{81 z8MzqDdnUd|J*L_@xJsFYj7OR#GdR9zcm+r9`iCqHd8$8Ay2_pm5x>(c*oBYL7a6cZ@o0G9G4in94hI zIgvV)QJBd=#(`9?y^Nl{q`XuGbP#3sGG(@(3qg!JU}84MHsYcYpyE@) z>r0g8MNwW3=YsZe38>DC_)<`9u8|XEUqo5$MNMCdd^cYmnGIMd7a5|M!oeJ*4B$s0 zKB|S26bzYsQMB!JH{9EBAHvZ&h%wzj`WUQVu_p-+-EcMLZB9BlGlElx1QGA*AYz`$ zkvbHZGnio8IbPO9ln}avRd>)%+L#Zc(-w5mA(Rstc%5`2a;W$26Op%iFaIJVI&r~v z3F+#+`&eX4@8!p7<8Ne6^pOR}vAFl{2QGpNXH52$2*pEJe)hu}@ z58`fB2GdUJ==-pidp+_W#p*azi7G;&58X#BfIQy8DH__J8}l(|2+)J~(I6Uq4T<=z z6>qRT%yK=}Y&^T)x~>gML0ps_P(Gd z*S=_0-u{z>zcYltQ-rGC48mOuEG0M*DCF>y%q+N~NU-4-#BA7QiPWtwepxcxJmc9e zA|>!EayujV49wO^6t@-m_Fle9GFz0=AqcsjrBY94M0>rbh~9Qkv z3J5_(7zmq(3}&-VjwaY_yWl${ej5Tuj0oaGX0s2eV?Uyf{5S&)*eN0C#UAR#Ba+RY zSb&qxE#BuiOEp7sH0lmsi_md4F#B6(U}|7QAfGspy{OWEQ13CXjl2&^ul*E|D|iia z2{Dx9HJCV%ZglmScR53V9uU0tHWKk0@tW(gu<`6p;yCNt-c8_A!-hw|Ze#_V2~LEG zgOyEJBD@a-)0HgTCx4Qq2oeoaHZGYTkF?{8OWk~(ykF@BFm=C@^N#OVYC?t|C!)IE zhg`v%j@L}XFoByFe)?N61CO#x&sA1aZe>_Ofbk&e$ZklwvV2}JQ2S}&Ks{%f-QR#DzBHp%ik#q<7yspU_^S3xv3l=T00KjnYWPx+s5S+zSNu1Uqu zbJ}}{pGFuaAJsNIlO`L;z83QtTrJw--}6zzJ5PAK@&kLag}1I{)V;&;Nd_(T$giGy za3Xc?S5KXlNImhZryiO}UG~*e4@;!3{pzWQCsNmc_0+U|DSIsR)l<{=NI3Pq!-;FZ zdIlpC84x>v^$adgWI%lT)icORWIzo4)ibyEnMhJP-j6Hh`4iJl(u)bN9-EqgjO&nC^&Dm<%(FGhq_ z!%O(Dioer_Z(ToJAxz+xjDihkNJ)po>YDsDE|cXieDC&R^Y$a(1#-lG2jnaq(f`%z zF}QV=9<_#H^jK&ZNsoHNXvmw};Yg6Bg(C^ccqhS@7Lp6_K4LiBtyG~qscsF6MRvI& zyKy4Bi6Xmc7m%Hlc}M(o6Pkq#f8Hsczd4Af$h%DBZ4-Idio6$INZy1lQ4S9&Tn@_y7f_BuG6JSN zhILP4bye9>Ysj=0!<_t4ykWN>NxzAWOq8ietjH=;A4Cf$R`JV`?gkd(KeVG)N z#bNk#l`h^e4m*<+md#;jlfp)F81(+Sv^fYH2K|aIXf!Ul!c`-kSruzfDrlPc-qMjz z<=8>@QyiI!y&~QEkG}^kKM%ebS1pkN-r5)^zBh&}Dqv3E_Hx(9S-2^@+U9wW@JF|% z(IE@90R6<5*C+Jv=ogsTll@y}0mHJP0A6#Jl)q!DwVN-n^#Gj+A9_$Z5^0HN_9z?y zU)U>z*2p*aKcOLlrnT31)t|Bb5k20$1Gl2#!29Y#*vj1Pb36^J%Y6f)FD~1cNr8K# z&oA3In2bM9{u`qIp}RZseRt-p_REPs)weZ$!H6%XSau z+M!+#6+A?&u(tX02MUNRwMsrx5FvJuHGHDrZHl!ytbFO(i%ec2?`h8OHS)d#zS%3w z$LlWFCt;L?H8(yskDY6%eZ>TExcPTiF-MAPUtyeAP)I1j;k4He$vxZ3DFboB-iL^IQUXePfZDW zihg(P?2@_b9C+AcIa!IW6_pg|K4nX>j@Q}*{|VG`F?-~>>b8b^#EPH_fsUL zx5g*vuWvu@{4@G4>HJqx$-HBH^SHQ{Zw+Fdbbb1cF|XSIt7MoJN0;p%5Otr~68#gM z#5r<$R5PMP|FCR7u0+4v2U?^DQQ%LOK_?G?2qcKds)BmgW1kzlNx7`b{}!Q)oh%+89KgEnIChdMqLcva6R_AvHNgj^PxNh%9(RY^Z8$^LmFo~1zDJE|&X42zS%&$<)vl11q>6u$yW;9?= zq|}u8qRag6#_mHn1*KN+ifq0L7(&-{gCX3RZ@M0P(b)Zd@FiNqL z{?d4BA?ENnZ%MvllTM81+Ia0ZN(NJPdR(?e^{h%6wVBJBZ08)b*9_>nnPbqX z4Q|Y|w9n15wBI%iAEEgubM3nZ{MugV`gp+F_X_b_duS$pL%vvhm;ujreKcV0y9CIi zaB%OoAukcL?t0==KwPvD4^9Iy}vLJ=*2Vu|pMU;5!sR59OW~$92q(|>$PcTdHXui$$i3xPabqUJ4wkk=Z&>l3%JdKX&uGUfP% zNX)YX+QB5ci>AB>1ryOE1RF6UbuWGBVHhl!7Sf>Cf(621&tDGmo;Dx(z?E72<(9wf z!GQzXg4NUMtDw`PS$sTz^thM?A{o&G^GSbx#U5NChf8P?CVZG0_C!B{M(EV#urG$G zXJVT0C(&`-{z6aBG+$`+yd{O6X*ZbbfcXUU98WLi2TVoS0Eg6&Js4gvBd%@&WiUqAQ&Gw<#Auq*gmZX)R(wr?x zce5wsry8Yh=R03DdR>c2U_7p|TV0HO6_D)_+L>tCPGe+fd?BWfPjPHB^%ify9;Vs5vG+ z0H1f&pgp@#Xc6-ZCK<3^bQG?9t-(2;H?YS( zPr2k4DihT)ErEyeB4#O?R~S34eJG!hl9%~&ViV@Boh-xcARPVtZ4Lxq$|0Qj1Us3Y zzvF`oBXE=w0Yq02lSp3Wr4KlX$M#o1dGN}yeeJU(Hi2!3PFZ@Y6Kn%=C9)-ENjuQL zKfhDTQptlo$?JOlpz&TWPhFZMPp#d$8)YX%D%ARlL19o1DvHC!{5m*Sz81#yCnR6d z6~ZL1C&^cVn{e`+{U<&WhM6x*d5zyq@%MN5eNz1WBYvMAe}9=q zuoaJ@`%h%LoG<`c&$c73e}S}e5@N&&Q0Cpf``O|Wo}X307REIK5>O`%pbY%2pX{3e z^lfyF!1R59^TCZK)QLX^S&%q03DoQw0hwnz5n`wy({&kWy$cZ2GFW^dY*j(7Yb{lv zpo|h;_B_swon!`t4CaKGH>&u4gl{&fnY;M!%dTI+f53IwuZ?+oK5zx_@&?IQXGwh6 z>MELK%-j|^Y$Exi+}PU-JgeQ&cRqiUJa5w^&7dzGS?v*Z_3ljI;_l)3x@D+TH2;@xN2c zU6=iYA2i9}i0(XypRRv_40ME0@*+3hPR=57Cg+AS9zK#f>xkWBa~ zVM@*;go%jjF#hgje7wj)4=!Iu$UZ{EXJntzdn|HzP$mYCd6^%&9$RGWJwzMHz3&r4 ze~JA(wUe0>m|LxW5{>p^k%HM0YnIjOyK=n(Ea6al*jNptaJ@jagC{CXT$(@yPJ}>} zjk${UCiQbw!G+0GDRWWWcp2wG682S9WJA^VrM2ygo~?6wV^$~>+UI0jhitQU^{Ant z*0y7~Nc)T9M+cMVi{o?*FMqeSZL@XAbJni+RothG5rLyv+xA#9j&YnliASDjJG@ru*-140q! zhltY?H1*Q?q^+}gf5F<0j~%SYCL$YhOe4T;L|rF9fbe_)T0N!_;3q_Yp^F4KlYtp~ ztZj2uYy0dhjx^gypJ>Af$n#5W4Tya|W*1yr@JtE1v!}eUH&)TM)z(&MZJVEEYg>$O zvaQVds}V4KVt(s5EWKsTY;DJELk@~6!EfGM@x^CHv&zV4`QkHM+d(d`%QobYwd)91 ziXseRXhp_L_O=zOH6K^=JB)N{RZ`8J7Kg5WHFTT8Gr)~PdMQ!ChsG02Zn18~wZK4N z+e%c@FDj{uN{&S(pItW@FBx0c72`p_t?Q;~zFCaezf$swwP)$EV(4G!K~+0}rf3yE zo~ZZq*y)`kKJdYS?#NKX0_i|Rn}4Zjs?R_;L>%cuE5>*Kw?~ewzi!* z`fZ)Q1@W#3e=ouRW$mR^sPoBtK)UQTb(Gul#GN{rTr1CO1U;m*|3b=N2YO2>AF3UJcKiQ<@@ca!rTk4=`OlL%)di4XTtxUf z?MGoL<^N7A-;VOF&%{~aFupou?fQ6_&9jwA*xL58EuZdp)Z@PkUsweFk2KhcZoZ@| z#f#zTg~t->1iyd}a0Sj0@WFP$2iq!&Plh4A4Ek9aO-hq7Dg9(Da_<=RgL`=Lxf2G| zPh!1^$%od=Sr~14KD`D5Y{x8>47zdQ#4Jn%xDN@Q#pE%XKeX@=#9!7S1M3p;y&h{U zN+xq96?K`p*5as@E>1FZ^bEqPsJ;G!tcvz0a$8XleTz3_ZF`f7y8?&WC|vNuiO@oX z_hiaIL8&J~vnlXnWJ56p`J4z9QQ)gGaFfjTDhiCsK!NBJA>1JKX3wJt#M9RHE)`4P zGlWcay}7fk(~4^*t@Jq?TU%2Gnp~S$fjo)Hi1Rg?Y8FQVg??)bmObFZI`v{-OP&iX*XIo#6pGE;+I237_+g> zg3mHryFM~`PEh*%;~2)ndyd?MaRCEhoc=00GHr z`X^k%9yNS_G6$e{tpnGRY)XOaNK^xV5he7@Bv}HtYrufg+72n=>q}YBiA?HBa@w|9 zX}113>Q4*<_Q4fqATKC_J8YvAe2QjZS;t&90-pyL{F?i;KQHam#wwK(r^kbzN+Y;r{=Dp}wt`jDL}x z4ejA?N2|L2wiOo`*3p?-gR}7%kvSNUzM z{4VCtff#l+V(C}18-J|Fpw=kq#2@QHtVv2>m~lFM_y~=sbTOD}ZyOqK3;JRGL;yOh z7+ep5(N|GF*BXrQ8B}Y=%eLYd3>z+`q8QYHD>%?*YugBFU}5aJw%c;BAAH|2x)Z2v zPuq5D+Y8qGt=5j)V%)e1XxvF^+!t@k?czlHq9>t}JUQZ~V->i33HYYA9kt~H_tdJ%ceA#w=-p@El)ru zJf1v&CSmpS&$*a=!YZwPXnk^PO_g=s)5O^dMsZ#r+_|OwrmA(FXV;xH8}9ihqUDZD z9$14%UFKSA#!``oO-U{J z-S*-w;TMkl9{DEd!`6|R19{Ze{(TU@G2J!-fp;R)V(@ z!vTV`lBhEldsj271F(BH`t9oNMcMQY)F`{%GP7{dNNB`-Kdbq4~zz!Wv?b-J3|YRIQH`JDA? zv#;LWG*oGJdc68r{(!I9A3`jO=517sdG&Z%>h=c5sm;!nYF>T*P-W;)BvGfR4Ngy! zo8YQ@xf7YGceuUvs;@x}xmSiz9WEm{{>IYrjg8GsCVnk&p}??rsn@r{t1Nfdg;2LC z0e8?HSniI$JM4~t+ljW+IXqr~LJ|)%uPe39yzJT=fuGiy-|c?o8HBe=e3mpc3@!n> z*T}d}D+<(1u{CSpVv6FoAnM;0Wj&rJ;m(#KF7yRZ>A~Stb1WOrV|XH7?2+iTH~#&` zA54Ou^6P|q2L5Lp{EN_DqcmpakY0W#N*i7y9m-jCRraz9qYvJ0H`h|X} zPZ4^qzpXQUm#&*a4vMLnE1q66bH&cN;-Ov{t{hj35$ramJ45>Ba~sE??Bs86UL zs1FG5globx;g;}8I3&Cgt_VMb6T&~?obXJzC43SN32%fet$#JT(E3&DPekFL)^3fC zG&&JjP5<}gd43td=(+x-wkH)IsrX98M=HKj@eyxdhWHJTzk%|1_RN=GeEjA!YtlU_ zJ=KBmLpUV-5Dp1HghP#vHTu=)R-;#qPc;6}_(J0ct$&ae|FnKceb>TMpQY1#V+Y~> zYWaHZlNr4+6ZQ+nmh{G!mG;ImXZ6P3fTi5maf8cxV`XqRl=sFq%$CG)Yo*IknS0{H{iNpM)f*e11!eBj=LD? z^5Lv-Cpvp$k9YOPj>Fx*xi|LUmfl!5+>A(X>;RN|4Su;d_A5NUzfI)-Iq=SP`O;rSE9e;RH$@+J2U z!rsIC!kxf5+<4?OA8sw&@8R~rorZf8uxylB40k2sliT=wZ_JLmoy#3X92e5G!~F?) z{P7uBgX|K|tp|EzAHuoue%UL?8?Fa_+iShC+ogLQ&x_y&$tU^G=P0h;`S4CoOGo)@ z{&Y`$dc9wZpZPlQ1gE|K4*n0|E`B2swn*pK;?Z-8&Uq2%$8gpNc&Qci>0Dy$b(#HlB(tgrl&#{^eAx70)N}JPkJk{%d|F+)wc&SFdvf z+XgrFUr)te`Q@ot@W83qI=KG{_ZZwZxHsWWz+Ljnsn}I;Cb;QvcDO}w&2X#XI^g~t z?g_XE+PvJc0pCSW7gbxm9-3Vc*D(3EJZw_DQe|ByG3<2VSQ5HqEe8@+2py)T|ZPA zA8ZMZuXiu^)VYJ?54oGid$G*oKY3v#uw0#?7OK}=qZ-5Bpr_I6uFpr%TZ4g*c>wI2X^Z_(QVAJ6b zc)TH*y;z8vOeU2Uh>i zBb2WiQ)1=k=W)N;%k2(}tzveiI(@oYsD!X?^#x2byskS67g3PJG%}iuY|aHY-*BD? z6N=6ifpOd5KX)z-Sc&Uu;qy}!xgB$>9P=tI)eC5yZCvhgugDj4=m|QSouPndrO_!v z>YQGzmHlpK$mkrWy84E?#;N+a&EY1>LPj3L^+c|rfYTfF`-1NJq?~=>5IWL_z(n4N zO}*`D4Ew^tcXw*O~|Cc!Z;0DQTDT%!9!nA1_q0$rvVV%X4gam56Q?*y&l@<2R-%<$45sez(KrtXpc#R~vi*b=DGRXtvF+dVL|))9e={2gOv2J2Y1GeWETP zIem#}0_tR5Bm;}M8X~Z0kw{A%rg6>Eaq2Q1i)q2GLT0k%x(3=6@g^Eprqt_T+M9q? zDoYtA$|iCZ?-RH#M42L|1YEo);3Dq?T)ZdXqQT-Fgw$WJ-K6!JGvIavp*?gkOQ!IB zuntp5z*v_UIU57eC4#Yeh;D2^LmV{a1!<;J$Y+`n4Ai01xrtg85&*LRh`OWu<0&LS zOHt>?+@S=qR+-a^8J*65*%rFynfS!=18Q=^pC)G9UAvy>Gs3^aO` zF?iUkYO$fy^Lga&TFoQA#O9D(oGq1YCksMJ5;l`esrGdI`CC z7}I1GA;Tne*hEoWAu8VEMPGZMCMOb6AwId>fpKah2AI4;A{~7kNi{N-fRTeYHrGqmY>#dn2OS4+qZ@so0sDY#kZPoc3HNo*|4A>%3P9ctdB zCP;9Jzl&tIV2nqMd=*k8A??P32@UFa`J~#cfVRCKz)%dxOmY&_kQ)0Ggd!%R4VX|G z93fxG+2nxN;qyDf!8@qc7{=lS4OJTaO1+0WN}a9#6RNIy-76q`EJdtPAncw>LG)hl z2_VQ;jer)~)>xr52HbA&rl)a9$kh~fD=U_GLTkrUyeU*x#70Hh7w_D>0Ry zb-pId4=6u4RpkK<{(!sA7XZA;>2fzs9}DCIH^KZ>3bs1#$Yb<1)h|+4snb@@$OHfD zLV;O5h~hLN&e*9cc02qY@6!KrZBV8%Yi?ytE&5eg8wEufmAnG8TLH%crmPlJQxA$X z)%vkhgJzg~_*+0Pm~JJ1(W{im1$)$ooBa+XQ+)oA3A5)4XP~}7d#6b?9uN#U1E6>H zmN^q0g<|m32boiYCiRUq$SbL7yRhVdlNH8zGDcoF7kz%G>o{p?2P>Bkgi z;U(Z>cL0>;YzjhrxJDhTF*6A()ZcxQEOob-!0%zV>UVkqk|9$@1~CF*U}*3GpKh-^ z;A|?u7}4k@0he=?QLfDkUMmL!+6>dASx{LMIM5ShImM|$cnEnQtWX>mwz`&zr51MF zu*KF-C1vh3#i4QL8J4Kz}CEpGoxR2KqA@e>{^F z53=xJ+Ao_|rC7?!$e9((tXWFUyi&zJ&!&`B+^W>vQcbQ_vCW+gfSsJ9dhTsCioM)Y zNdrCPUZuLcs@zg5AC6MXtT{?eg>9bjRoiPEm2<1@isli)D8XSzqPg>G9oWDuQ)*^e zY~@N#d95<5Y5_3;WmsdiV8Xn$qWm_cs=V4hueMyNt+rIwRL!j^S7w*jin?eXk#kvv zWi~2rnT`Bw<~S-UZ_>TpUS2&{shN9IEr9dm9$C}5D61NO#48q(r4`C7o5fy5)w$JS zE)&n9GNo$nTpPXf6Yn?KEHze#rM9+GdT+W(7bfz%t)dbew!kHM=2f!SZqZhH#IsC)_xl(EJH5&2Z&RzeplAL2H{T=UBFacv9 zm>_c18TSc^9Iru7Q#k$-XbwUm3^a2J>5l{FU$+1SFIQSj3ulPR<)bnSXS67^!W4xF zITYE-6$^#Nmn-zr=ngpoz7;{eKVCw;$H~p0C)T~er5HdOC}twyE_c93Awge52vN9g znm?WoEjcJsV(EbwaKFn#p><8pWyjHwxBNIV93Yl)_nTVaV_KSLY#mv7<6FR zLUURiO?Z)M`-IC1P!@929=b0KR?ZOQWMNvMXv1rKFjRRgMR*V@98aAG1*y>J_yR3F zuF-^xX1if%01*k+12RXqns-A%QG;qP#Q?|)J#j_XVvI+yGaUTJ(hDhtQFV<0CUjxWLz>7h1(MxuYUNgX6MVKjJ0{7o+HOsZHh z2*f2KOG#0u52=KaN)4~391vMj^XU_jilc}SaS9WQR8fq0k8A88%V3qOX&){i3i33Y zW+ak{u14?abXlTYij&ZS=)hVjB7)SC=BZvO8+l!32uq2#qNKM8x;N-hNy`MJv(J1^ z!lnv9vwCgtYml~3(59$Pb-LB-NrmeG6JR+BrEeN7Dg;6=w8m3ZzgTuVpdEICv7!5J zad`ctgkeLVgeo0bYI?u=Ak?*JZVCw#s{2JU$A{3y@%wfUhUp~YKy*nNoFJhM6|76W z4iz>PMv@@MNzrf+aGXlTDqa_NrBo72?JNTO2~D396?q|r#cLjCNZxd~2~t^dJ-8@M zm7G$WBp8*P;EBXO1`R7(xlie#qiX~-6JDO6{IMW$eNXf8CF^RJ`Y2(VaD zB*10?HcN1kV6J%TA^|Rv;0Y<<2?9*&S4O6nm*hsKhYPCJd8$=-o_ZCYr(%VvH6v6) z7;*-j%|X))PiP4@NT54)pF@(czKjqsz#-HbE%WtS(Ij5}O1 z4`WXU0tO}L#k)K)mO|50=k|vjP=z?m#&jcYaxBqsE+%PQNRGuF$C3Kwb6rvW!YTQ= zM+!l(5^EA!5^-i7;>ua2(!Ll3k8+A!D`#YTckmX53Ny>)UdP9-c(A0M#X zV)#UxsohL#XeRj?YwVM#(PJhtydu)L5j*0Z+WU17Z}CY zL)hVM2JQojLplS|3q?x3QV%x*(S|$>SUEySGx@3m^!j`8`=N9RxwB=j__>#+sPj^Wr z#6y$v)p844KETO2jmfaDDjf@vrW9ws0ZL#fN&EiwL(y6U zO_We?P9g2y4Ayj#_YUa6vE&t03;7?)O{x4OKPxt&=p3tQCTXRG)xX=V)MLr%hJfvZ zZaZDpeUdYZF%rqvH9%1E>vt=m$gp5<^fpU%Wku!eDeCxhTOW`H%+1E=&{T((;BARB zsJfurg*3``;JhfrdJMWamDI{mAHZFt7knk|TUw$4o0;FI0G zqAZ5%gi~Q9(h7GFt^@`yo1~G;Vi>w~!l^KlX@xroR|4bHO>k96N47xpnG!$TZn#3E z+X#0It_tZMh08`7KiqD(LZsUWcMLA|r`f3@8$}kcAc;azHCtgyLooUgjDC8UvKj#t zfsA*Mk&1`r&IKz$Jn^fVdHUhwd+n|&!cQv;}ZC!K4ID+K@ROmKprR%;DHMicyyE8DZAHaR(5*RXm5yz; zP+VDqHF3b}Y^t7PzEMP=?K26jHk)~?Ko}5tJmnK|9fid_iqU$fYDrZ?b2hK2R!Z^+Ap|IvjN?SDMY{B9}AhsdEHju;Xp4sH+K?&2>whFklKeJ)vMlU3rZY zf6B{iERL!-U{7xX*^M{ZzQQaCLBt z;g-M!;nu*l!!^MWpWKu1{}S#Ga8bBDaPPpKf*TI_ShySE=D;n6n*w(OTru2qI15}k z9Q~-x8gI~3bN$&Nl6}8+UPC+MfFk+a2GNR%iKw8??^niPiO}5agw+@}yXuw(3wZW4 zl44E2Z$$K_9iq}vpy<8i{i&tP9d*vSB_MTZOyb%Swpqy_ z2T$ylb>gFH(f~o=11e`drbI+d3oa6?v`r_BDzD9}mu=R! zPmAWk?jhNnplQG=VNyUuMJ?h{s;{t2DRvp54R7#;VIYAzB^oSh6xZ8@gM<%fQl(WQ zDTHYxB8eH@(SjZFAoeCHOQ_KU6r~BOzHw?82z9F1dQ=hGFri3SqR7r~3lz_L<5eycP))m7Egks+6 zR@f#M+jf{+3hJEL96_fExCq23at92VpMa)wbm2 z@mMkhOOv-u`8Dzu73ZV0Y4ZO1^%i-vbxvyDu-3YDJ`hVLCgg>lgrAelI-ld#QhP-L z;JCH2l3y#!=fcz)@vH5Qn_w3YFBI6b_*n(x=UX}GHcO4&QBz?rqwrGfl+zPN_)vnW z%B?kAIy8s0wI}Vgf}#2;Q-rN4ma~CI>YpV}Z=3}%9gj(wp0%^YbavS3ZKT{~Lah=XJri~y&0#OwX$ZkmP<1lZ zGb1O+J)Va%7z`7Isy7yv$~lP!NgS5;sOoGs{_l$ZhZ zB>(iu8<#m#0|R6k;BfoFjfe^BCawAn4I-+psD^-h8Fr&F1&O$Fw&!3T$SD}6PSpMe z-WZJ60GzH3){I*rHz7K3$a4Z~7%<@lVVZ*x4Ac|ARHBI@#-Q7+E(wMF!71a%V{}~- zb`@Z9!l8C|pwa2|Egv739>@E`O-|P51;YoM-^soDgJ|g*^eV8PSH6kyYdM2|o?| z7)L^G%;nb-;;vPVK1ddtNhb?QussyB17@hWG{nLOs0B*}zeX#pnr={`VC|bpKT*FI z;;4RzEZ-F*)7VI1MXFJRac7IL397LP)udfYveYDnbF(iPYHCqw!%%u}A%n1l->mmz zFUE%nR3Dp+#yuc7 zr6h)(TAzp_EF~h@vPqj1x|;%*USQNAD{$q;QpG%>RGCm%s@yQCRKb|56o6Z?%nOG+ zO$AF}p#)(@JL$nM)6|MeVG>5KnEAw6AqA2k)*ZsWo%V$Uw~-w=LHeB}c#43HiHUYx zRC$WY=xm@h75a6#2ghd6ZxoGY1jH0(!1+9Ht_N#X{I5DO>AWq`6m4YENu7j8|1+yc>8E7QJbs^X& zf|LkrZ0js!QCb6I0Hu`~Ek1EQ2-pXvJCLPV2oH*h7?#>NHU?q2DO4adD-2C}HO(M{ zQabac9rPmG!1Bs6$E>P(Vx*2=ArdzN5|%! zQvgk>cjQgHD?Fb5EP_rQrtPp{hFLOn&+^C{`2|T}l$cDG(e_fhSiBZVHn&%8jPe_{V$mL})^PEHv652KATG;^ z&#zO|pmRAjMij@@#U2TIW>Q$s1406X&ZTbY6xHGIszw-#!#WJjChITaeVU%3<3ZC?9^?f<|ElaVvLR8AZ_{7dJWOSE-r3s1`1`dHV$`>2Un3MQ&T9&qOh4H zEoQKX$kxET%%Y*EP~R{xW|Jn_>ae1%x&&Idr%AZc#gH_@T7%#cUl@}dkuiueFKmi0 ztZ}|2jFnxTuh9#{XvZ!Dr;u>zL)EeQw?Z>sX01_PQ&U*I4lD7du)@<}!W<|uKsGz; z!7y%PLBTlHZYjIfaa)DWR#{$NmTWx~mzev+ueMZ{&9x`TPSZa0an+9q#A60&UY)HSFhM#d#|j_S8^+S{TAFdAypRgxam*N< zL0fEByb`BfMal^xrQ`Z|NKqmecttc4FEziI9#UpRIZe`NxB97MoIDO(e~0u|*71z^9lR5J*y>BGu$6;~5KxLYr?gc~KZv0Cije zpWn1pG1VbC=_6@=g%@jo`~{Djh#yT@@zX0tMiZ^ySz(+MzzceW2wBhsT{B*Q1CyVg z*pZ6O|$gK-%_P`1r zYeJ;Zyr!mr{*BK2pm`0ACa=2@GyZZU=X>*HsI0vUuP!o5*RZhE?WT&?Ey1z{b#I1W zOid=j4wURpY7wc%^@h6Q7c!ShOa=v(X<08ll%gF5X7MtydowpQA~DAT2%5bi7oaT{RCD>0};M zuPutlsmaN4Fpc-RG?^F&sIa<#uo};n)Gc%`p;}#TDYKUsGz*!TPF|4$j5bT~pG`<; zil;JJsVFrzL#ZW5PE`hy8xtAF#RYW*CjF$EV`@Lu{a%_bvrl#fJP+^2ZWD5bBGRA5MmsxUIULTr-%?WfI={WGD3rp~j2ZBu| z$ek|SP7&ub;zz8dL86H;@NfW1|CQMG0CWisjXHxXoQ-(9)A~pJ^-^4ez~cf@Ffkd z@uM|?#xR{G)Tx#eN0C|BbMwwDmLgED^o^P_j+3T{cr>BP1o7$y5?)2oEeS)F#Kod_ z2>uK%g0#<^8pMq*s%lz7j0~jurDm2NSV=%PMv6{`G-J~TeagBq>IY8fp@jNUDJ8Zt zP?J8D;RO1~Vnj=yfQ#a7X|X=T;HCyhyQw-?I9o*dbayG$Kx#d>;_>oDAdM+B6ys(d zFa!@}mL}|=f#GQbnY711LFzEdIg#FCMUZzNF-p*O9ZqyY(}QVorjaa|Z8ZiiMkEg+d1&vSz(O!%X(mPjHFMfjbo)K?`o`P+rd_8pl?Vrl4Ab)gz&bq#u4!)#}RG7^v}T zlv=PUSZj%mYr(k!O0+UfEfl>PQY$GJDLJW?Tw!tIpH5e4;ixImMtzu!d3f>&Lr5UP zNtDG$chPiz9QhhYp0@+#<}TeC#^WTtSYBLf9TfR2_REeSgRa7f=N?T)H8R_(6L? zA^gB+T<9IAF2r>P3t^$Z5SJ1x91FKl(zvLIkObA6=<*3wD%5;J8K-5;dUGa#2BLLV zqz7QQrNalEJnbi2k~yi~9$2twga!Syb2%!f>vr1SPoM#cN38<1V#nytWY)L+6p}pZ zF+V4dK_CDhjtvTxPC&?2QZf4!kcK^(OoHWG;R3TBV5SeV^)%+C4~*>+md=!lt=ye1fYV^9)j-vnjXi5#`X zju<~!70ZK1f;z-GzN8FA4la>HVyQrR@SD(z(r6_chX?UCb!=il&_*5FVCA8VJ(5To z1HLe>)`C9TjctXv0thA}boI7stSz@!jTtkB$F|AL%{H6UZowuLwoRZIjgO;RDYRS5 zrRZ04s4-qa@_}ZRA|z#NcAF;Yb56vCl74VkwHjG9tGxxajLnXzkC|d$apP;2YVr8NruIVmKXSpj$yG(c9$Mi=g3$r zSCcZws>|7g9U-}t?KgY5V{^G zlI%g@tdo|gmPN^>v?wc{nE=M;9G*u-`8tA1-wX^qNvm?oU1lk)|Fj6O(Baj+zV@7L zVSl~aM*)C*ld_FR_-grTY66LHCKn8Gs$FUl;yub^PzqVe*IL}?G-sSz8Ln$`hcSZJ z61Ck#)JD^wO_E5hBp+ek6bSNmlw3j6bc#}GBdPx)VDY5m=F~o>kxi@cI65sAe^ZMJ zq^P7xcvF69g{6ijkHk{0FFZZ2Ao!ESD`xm6&r)}jXNk`zK5syz ztdio`$-*E+YMN$Aq0rB%oH*fn^J1EI}_PETO^Ti5u`tjsgt!SqOf`&i^K8x0k-Y5A{J$d&P}L#6ZPYcf}OG4BBx3= zU^|`qPFe}z=BDe#%}t51P21S!h9@%9`$#sYZO&$BTX1*B6iDo0?1ht35L>*s$5v)f z{lW%BGzgOk4{S2R0lq~f&Z(e%><5#Dv}g#ar?tUME0e;o|AU1%R1Ar}Npx{Sb)_Xy zNGITuiSMg>B|o|r{D{KUyPqrC(JmQ7_v zR4UcFsmxz&(zu{RJv?(9)KZk6g>kCe};SuAI7!u z9Ha64q#;lG=-kk2oE!cPL;~eMr>6e7-0EM{Aob5PSpAERQUAO{)IVRW`e($$!0z8U zET20VaB0{l5*pzcNa;5Hj1MM6K;wH&kZe{^)!U(0iRiI=7QGbQl;e^LyvyhuUtsC$W38p|OKJ{AGsnnJ7%_srSbA zZ8w#hTl?Ofh`}fxWn2o7FDRi^mrKvZS5C%``|Y5{(IWz5aekWOG*BlG}0Z$S_7z&c)(O>fX%=Xpyyra1!e%(0gHf}fo;H6!W|(!4Ieoy`Vi>@=6?b^ z151Imzz}ddux1zRgAXAVeUAJB`ddPJ5wQ77$OGGewZIT?3ozp=s7y?!Ti&|kXU>h(*JkXm2J-b7CJ}?Kk6qpYz1Ny%U=?%aja0j^q_Y&?2 z=~?(lV-c_bSPCoww*4FOzz}c;(DOq`Zv*DEBR=?2Q9dvSSOi=QZ2mE%*8tmqo5}qr z#EW=fU<~~I3^`yUa0M`9FZ2WJfo*jC3(8Lb{((in(tXGWV9u|w6A+(v)VqN-od|a< z($fV!z$)N&y6#4L=o%P6CG__oJYW@Y1zqd_7hSvXZNUk+24(^M?hbt!T?5wvGotX#G++(T zI}!OXs6$T&HUo=+MV=153fKm00hUL1=qZ!n9)s|JZNOE)njszfc3?fQh1`JxAMC1$ z#dqm|jliYAHsC6tC$2-^2n+(Z19O0TfcZdi5afo!A20+g0G7w&dw)RxunxTum@&LV z_Z$qpBOnLN_v5>Iz|hDJeFv~+6w-YN{3YQVeZZOnJML%^Ct;C?9d zq;%+Iz~E%0kFJ5Oz{b=L9q*JAZNMC02v`909145VHL#wpry!odHXvRDD5?(c&np9(u127gC(=nH}6N8wv|K>u{a7g%~U!aE#%7VHl!orU}a2Js!sW?&7_ zl@7ZA{lKDZgaa%EmI8xw5g%X;u$B0E(02s%&xd`0LEtiAc@FZ0uJQdy*HqXQ=m*vx z1AV}%V^Q9KHNefl{NoVsBVm{05pQ4-umqTq*P%BMo`~|80r!*ey+B~b$;c;Q6>uFe z1Z)BpEyQ;gfx$}=?ljn~81@3@Uxs`K76FS1uR^%M`oF?1gjXZ~rz1Vrz@4xRb_0fh zyMQ$nu>TCCb2Yv%mO$@#@2+a3&>f3?MLp${rVB@e(JtYh7BRlmhV9_Xi(+^larcS7k$59!of36JX3(`Uoqv`&2m&@;VLZvgs%JAgr88!!Xd4a@wrOEJ+O2p@`J8tAwPimd9W9-2pBgH@jDUr0_GP$Kd=b6 zkodEqpRUh=eqbrEg09bneqeba^aHDa+krLbK|e5ONvFOSSbBb^?wODH7Io@Lz>Ev= z%|~GA#hrRKu;a^U|trUHVdBQ5p0Bb1J%Y?+LK?>Mp$mSii1I-v!Kn9D4Ggrw)36rS)BU zBjKxEdcui_*FU@T0$^|(+<`^FDqz))F1-!t`JzkDI0<&qyYzBk&d6@P8R!|)tp`sA zKdxIZ0hTW8*0%r~&*|0^79!m9yY(!fzoc7V1@x@z)^`C*pYPUFPC+=ky7ekx&ez>~ zD==esx1P2L@zA^VWkA2^(Hntnz=Tr~pCLVZ8L(}1kKP2#7~7+#=YyXB9#{jc1U3U} zfNj9dz!0zr=$Y7~w*vjZ5U>;&I1Tm(rUR>hEf*kO2leP_7ee3U9(@JSe^`%R5BGZD zUXsh`(X&oR`eyX#tAL)&9=#oMrE_}p{EHB;;}CD~rT8{yH?RiZ=iCUs4d3V#OCgVM zbC#U}y}(LfK5!xUsxx}@b->1R5fAWf!1Tq4KQIT_yrf5uyBPW}fIs4wLJnAeNsnF# zzNiH81Xfk{=rhg)56lNvt%f~-ZNP#{5TASDALw6;cmkV&!9O7$_rV^QB7QIT=q13S z<{rJS81Vr%1N|RE?kwaFa0ajum=6pAyWyVGg7jSu|GRMwECSZUy&TvCYy*aXRlg!V z1@H$fyaIBe9=#aXrNFhguIhr_aa{wf0yY9S0-J#iz&79xUvG_#66hV%tG59CV|(>I z=OTUMdiBby;BP{&zPJ!_z*1lha22o-SOaVZ7M~CIgM0PO!1|P4Jx~O@9oDNC0gI;g z>NzM!jleQs@JL)EzpH>Xzz}dRU1#*_NyxWGpcna+F}+uB!gUd_mGTwnMZSfA-Ehx8 z8hUWu2y8_DRn6?xm*Kh%SON5BK_2;3KdVj&)jj7y z4jBA1?0X^nZ1~8!%_9;fZfS${H^>x7fE0B+HF9p^Di%JmR zCBQ2YF0lM6qz_mFY=eB;GWf%_|L<`B3;bP;bmKastXFRa2CqZ@gU??MI|9pB^yLw$w%vnr0L)m8^euzjy+|LhZ4LAT8`mPgfI0U=@81#b14u8htr~jEaJ{}) zPrn*^fq`qG_n+W#UH^Ws9s=g?fPe5sAEG>6gZS)3c?M=Q!yeF64=jOu{>R84@(;{` zzl={%FMz>M5gyRL3+W_(z})NL@3UTgEwJ%(*a=wk1?;pO@n}K#kZb$8SN9`+p>I%6 z!25TjJ^+i_kUsEL-@zWZuK5Z1umb6G=z1wI=+gBqzzkqBun4#p*c`9xS=U2ugs!gw z<^!vN<-pBA|AD&hS&96dsOt-X<-j6fJ#YoEDpl8;fH_C#deRN>4@?7=19O4(3v|5- z7&=kcTY;XFbUg&j049{f{R~|X0z+r%`V63_K-cqt`RC|*CApu6_yWs|p#Mgs>mpq* z0QxW0^%}aq4Cw%RE=N3X!Zk1pSagN1F9ntY%Yfy;N^-v%@c>p~4ld(n_`{rB8IXz?=yYe2`x>9ir^Mkp3LP5wm@9vwi+qzQA(Vt=er)?L3Fi zFFDUizQE&$d5Fg(@;A#Dx7>NQkD^ZL zqr>snyj+?3L2KT7T!>MI4f5bd=`Dl}&W^(ROcxpv_(yza`vS9l!P&mFS-y<SPD0pp zkY5+h3my#VaFPd=2|u{S;ARn%?F%e}#fYbO^j2DU(vbsxJ9szA;~;J^xJGc4ZaOG0 ziotCGhi*#RvdHJj_CdH1m#g6B`A0}k)Rfx*JD0h_?W|36+8eseb&j3GTwj?>#%nLa zX;>H1V<-+dD6Sraxfz^MCMe&Nz%_zH7b^1z#fr+QcB|8Q4xOSPc*t)yjp~(cJLZFs=#QI*VK@ zxM~X*0=L$}d0l8DEL;FwB{(B4l!xiyD1IkUxG=++R5FnmN>3i#w!qEEbK(lYHGuPy zJPzVY!EFY&L^6t71;uENYH-QmCNq}{tOJ(?ZVS++K4$yM9Hw$|g3GQrkeDiy zE)?2^M??Cf6ltVnxl^l&LV2fn2H>{vDRbVYgDU_RggYH1mjiAYIDkBeTa1fR3s(%T z#KNrrR}7AlLkIb-0=E?0U6N7UMsT4pzbSBQ0Jj(1a`J3#OApeFh7dJawaN7E?1h-e zPb9ra>nU8XJxKdv@Z*SwH1br|_W<}3@Ura$e-85*;8%gCI6yw_eDKLQ%>%y2F<_tOYJDO*DN6E@-eN%~ZO&j<^r z8-8lu2*}=6zSgs zUCmg#ipDt|#L?TycYrg(r2I+%*96We7ky!JTGcQ|%X5S!R*rDel+K0lpSQg~9IB&5 z;BvvmQS#H;ciX2k+m~lbr#nK|pGRTsfG&YGH@mPX&3oD1NHf{lgTX*I z*745gy3m(qdpjTaTXtRuU4f?l^i!Bc;F7?Fhq(-#AKd>&-Zny)_kDAiyTEzCg@@S= z&IRs&Bb_tQp$9+c50mOi9=K$17vh}zBQ;juk9x8Se)Hk?x9SOnQ3tu&KNv2# z7=8?RW?;SckECNE{1(ISZ>59k*)qsALJqgY18kLzirwF=oKH6zrj~{LZ-##l){-~a z`p*mVUopUc0DijRzZCv&ApfY7=*H2k=JD-HnrwRm7;v^i<_(ifQJ)OeoovWd?CkG* zey8r78maD(t_tYd`_cbg7nPG;(3SsjNWX~U-q+USzRX+}PcvH<>4CT(1NT+%i|Hk! zy$ZkQv$VU;0=s568D0yZXAAWFR#^x_t{ienSpQ$vA0FQmU=vbKc$L;v0e~gqw=?B*Y4#m_cJEpO4CA+^GDNW|F@Z1RYkz?T&ivMo=uqZCH z50BhOb6ifO4CiqdCRkroppX$n9a24Q|Q-FPSWx%-LGHd@Cgm@YzPFmsjwtm0dPv7QrUT;jd zsCnMSkfptHG?yAV9POs?`zSTbi}nCs*1~`Ad$xlmo~H4k=64Jj;K53yqXjxjurKdc zvgL2;7#mSXvLE#fd-?9M)e$z%)DE`Okw@_;f{vvBg!G4gSI5+dIyOQ_Ira?hg1+BO zi}Nl!bI7vQ4jsuqV*bXyY-Rh3!|yYl|Fmx}ilwd@Bhep0S6n1rrgqHvroFBLnb+me zwfQH^AO46zurlQyS)ugTcY`R7Qw%r@_`*QIRU4PE&i zA^kl{-+;OiR<4~l+1h*tyk(6-djcKu9xQwuFiXt~@siN_VQq;qRU+>*%HY2c`%*Vk z9K!wE&O12Q+ogx%(Fi@epy!Xq!;^&hnXdm?JTedu>U6e1&wqR0Onz6vZwP+v?q|bp zX738KC7MgsXEeg^((eEHc#O(x0u~?|pz8~q??;}zW|t>4wxeMV?TcR66Vk`9E}B=T zFiOEM27gtga%`#-+92Diib_FYY=N$dUUL}4?*dPImT}7`({UWaCVnr=hx0SQdoc(o zhy0f;Z@tIhvV2OI{rk!?k1YJn2e%nql*txqjA4%Bnuu{+3mxUy6JB7|q1pz^oT&3h`*Nt# zOZ(*4Vn6wf_PQ)>xfT^6y;)Rt(*qb+U@!W$`=`qjNf+6v47!rBZ~ZK5pJmNUnr=2t zf~m};a<>IKvarY9>%>K0c|f@%3vp4Y=F!x+xdHhZxY{&RdCefO4#OaIvM%Kot< zm8mMou7xa?>*PUp-w3V>+<6r+e_s46%ea21Z@}kyIs^B+i zm^p46!O^o7rTZ5*Z>0REybeKEZbFCtF~u$1{^oJZ>ae(RYmNE>FB!?0Z-D>sK7{g> zo^xph|EPT&Ep3)|c!W6Sk*>ASm4jzTzQwsIj`n4wI#L-)g7-FxFLXt^=k`oc?%Vb+ z-V1*#-V33t7|*)=w14s57%ARlgKFqX#`7`*`Vo5@Y>uQW1-kY?R~vMFNAa@O&l`>Q zhk99*GyIY!qJA9Ep)bNY*?+mSP|iyQahU1wDV*+R$$@T`w9 z58vOnUlHE53m0r(0og*xK98}vO`gy9HJJByOwoLej@iB@{F?D=aV&s@w$BYV8*Db% zW0-Z`&!iOj#fhskAtJB-7OrL%d$@pAlZ5jnTwte{495^y5-#wrjlkLA0zKgu_uFSw z0~J5Aebpq`AI}ecigU7qv()FAi%mAvuGGP;1#SySI8uA}w7I=QwU%>S?eM!7&m%>V zUK~`$T*#z$aKE^45#AQ4GG?u_ZJK@D^Qeqbr@9W$H7#IWE1h$OL>&uwAYaDv)^RfL1-K-v?UeT^3COQv_)WlbTI;Ccw6nE2 zy+=8GrOs2aQ)#xZ9KQ<3EMFC{8owHp+*JjCIvcOhvm7}e$Mv{NqQnRW=u z>zR1o3~Q&h`DfXC*5$QRHB0V2qcKPs>AMim3F4d%ihCuvvWq%&M>H<_>bG(45VkI? zjrN$w#3*JMy=pI7hRUGP{fRx$n{j1_{yNSn?y_#qH*S}l&!J+j!t(?iGO^vcH%t#ELgDrl$4^2kB1$7ub$xO7~Cyoxi2Ofb_r9p1#Yl3?z-55rqAR%yaOoCq9k?_LN6%lU zSoCcN7X*jpZh6GvZwolmhbgZ%kgR24;Zqp`kl@_^HaMczr1a2cZ zV;o3v@*alqzJ&{bTW8_Y!I8djzd7L6TDZmFD1PB`#o#D@+#E_fuK-tJ;i|wh&S;Ig_p^l>x>q5tq7I6Ow^P2%~ zb5f^1Jj`z{xW)s`eha{D0T&%6R|2kKbf>;1OkX*;Ts%*o7AChATv4D?|5F%O3$6~& z+vkUITfm9&o%)fS8FblYdam zq3^j;-gcd|AAgUPGXL#gr8(T!{)B~{-?$kJ_Z5CVQ0G9`fijm!ooe^D+HoH4OP!w} zWfsZ5*hvn%o+JOB=kv_*#hoqxULgN2kTV4;6cyX0k|}~&OQvGJJf&0DW-A-_-9H(9 zUyIW;9y}q+uJ6nCwK%ovNp?;cdQ!Gizu1DX7GBb+zk>O1(^$d?tHBhO^Fcd(*}eut zU)qr<&$o2y&ku{Bugt7(wY|PFLth#6r9arI_YR7n4_n>Co-NakvmL9!7AU(kwrqmF z)|yWJZ!r<{nbVtZAH4=A*)9nK0e@Ymes!cViaDR0d+n9Uyr8%(g}$6uJN3ipz9xKJ zXXHh-$#xOqR_!ER_0TnATc>_08FHX5%bdk2_A$#-Hgg??cKU6+JA>K}J6+~`y0h$* z(H$MdD-XK1GF$Y2E%$SDx^G{K)BE`+DD?$cTIFOBE1#}g+ z;9VtF`TgA=KMh)V{c*P2865SSf+l@&(=k55dpmYS zsy`K`c?0LS_WCNwETpdh`jUU`)R$P(rpi2-GdzA;z_xzRf|$}*3w_z0o%$_)C?A-I z^L@4E^e(rLpDgMWKU&yY5#Obs6RCZ)=)3xN^yNX{I=r8xg~laz@hdYwE4#~HUl|qA zLj0*PS3&E>Z>-TS9`~{4$SpcYtA?9f|>;v3dfU)_UV`|{q;y~lX?Eh`J|n) zxtM;^p0b?|mAf#MMZ7IV(6+lzf0e1pH|CWEA5R4-MrLQ16_-Y zyY$78%7M8rI>#=1Xv9F(W-oLV|D{XM#yOR#x+pnCtJb#mO@|A)-L^d9$Q<+eVE`g z+b_@dLCjeYQ~ma4=xfG1i4Kj_wwmXmmPaV71(a6LEZon$)1@DYdqR8LnZ_pCwu9~S zq88&3(w7f?8}at%T;Klbn_`<6FbVVpp`wQL`MP!4v8cS|we^))mE(5|U>7mfyB&~? z8`iBicx}VBmeXtOQ&KJ4tH5mZ4|t!`_lMf*Li=Ja1D{1OatoJ%V(3dgpvuW^y|c}!(~v2C41|KQP%8xtWYYQD&>DRWT2R^lC3 zk=ojJQ(qLJ4Qkg&S1EKYzNK4li&S>abs@s|saB4~=v{B(_HO;(m>;((yOy!Ac2$CX zI^~*s92&)SHQjytVJvmjvQNml!cG!;WFc_nGfgw2U}=h1E`s|Dd9f^u5`w z&%w1#{#a~E`@wCxjk&(8Y&#yl$<~bsqr0(NKMD21CJf7(h%xJGcf;Lyy-gXVyCk-pD*)E))3j@Gvy=^A6Z zgZXtaaz9bRd?B*_OMi49lM8?O&E4vq4fEw3d|!B&U3s0R-T5^){jaNmKgz#y_}vV@ zpHcz__T2;4gkEvm6$Yhi2Xy6qj`lxN+hQKWCD;y`F*>6(1rWI{Uv}#c;=F$?(z3T| zxp_?;w{7y>2aDmi=%;QyoyvWm{pl%F^?I9ePSdM;q#Y{Yr(|!pK4JgrN~~>ngWzQT zLs#>sJ^$7Ff~YJdEWrG6uel5d!7T%a=|Opr{u$ufp#NlI`qHNo7pZ)a{u20W8H9K0 zkv|-`#S!J;x}ocCB9t!mY?XPXCVV*yD>Yf?K zBiWL41&_h}zPCqzdjHDQYTGgeJyfPjpert}M}LYca9>$B$`o(ieKxF2q5e}!lOSFT zza_BKIGmH;MgC9k(dSTHkZ1jK%zcm2%C@Nxas_Blu&gN$@>>dSnZcR-E-_}L zWqPV1x58+PkY49pIgKi9LGKE!Il4!`lC+|+bYkiiTf*njS^|}eU2yl{-HV0fZVCer zd1LD-y}t;}U$${hWit+g*Mu*7^gQx|gKVA*E)U#zBBaeR_scd9g35+_;F})(Tyj@s zx4)q~&1h&ihwt>~{k>V0N#rgANgLwP}bdQ4?ePROOrq9!)WR3yH# z?VBN$09CorRdYj+dWRIo;-?~D%IiY#wct;p@DL|BEk|BanMJA!O`Dxx%=G;NmP?5$lTylUv66!nq1?F8Do|J9ml;?HdW;U8K{(xO#B8 z_w~1TB)1)04!G2?Fj~MBf-}-acG(NA0Nf6eGuMl^jJl8or=$~Lhx>c<*@)dDj(=k<0e@i@$MhEQGA!dt-P+M@7-Dj;3~isw)N;oMryN7 z^T*om2<0(vv0cf)AWEhmVPyTQN2gm|bNVkdk}uooq&&2jkjtU;t9m`4u4)trST7W{;F$Xe}g%$b^P5OEJNrL_gRN`6K; zW?gd6Q?7a{z}ZDRo>V*F!Jsqg*B;BWVCHtoi6>velbY*GbK>O@;gq~rUkF`+jvgIL zPlnwfqs9eG`Zg!1QHWX^zNkta*cl(b75bfvg^^@*fQ)&-)Y`<~-kj-02mL+y4p+#2Av8|PTh=XW%j%g!~l@kZ^tF-pIs z=$tIaa#VLJp|8N#t9Ot-#8usc8~S*Bgs0&83LcM-$YxFOn;hG#$J>`Di~a4FL#Q57 znv)jN^B@D$O!mrv+l&#twqpV+OZjl$nAoeoO!2@$aa#&5e`IgueQ)M^y(L0Xm-7_$ z(6#u$Uj0z}bXw;;qJK-53ms&^%jkEh6YH~g-x$EEYpU(AaJDb60N(SUYs+isgZ8ie z)+5F17`yfx`m^z?Ll_lXdh{audqvZLrBfTQyg0{)LPq(~4L$X5_2@_2r`u9iw1928 zk5zRwXA$yuOs{^6eSTPVT@)eRvffuhR~Fu>{m%aBdgQlswLsUF@xA)rBISp9j(Cx6 zdxRH6q`_V(r=mXK9o|<^y!ytuG!jtvv)D!$z&(I;@xTw&g97-;p46)^p!N_4wHKv` zQ|;z%y%BM89X@V<%G^~E;+2;Nee^E;v{$BvV{tgXHQ&rQ5!o0+q)w_DAtCL!KQ zuXpS8?oLu%U`)P!*Ol^bh)&Si4 zXdl{6i{$vEo!SwOA9F0vqr-ANU>gqY#xm8PD(mmre>?i-BTc6~CzL*I5D$KUrY`Z>`gSBnj=}5Ahv~dl>YUt?J7g2qAGhkm9 zEn&4sXcP<^cSC2cqerK;F4qQIodecRKee?nJq)DELf+}P|BLF8>u;_*Bj~iOS6^bg z=t-;6O6NN0totYK;VJEZ4jS#wz;QShcKZhe^2R>RdAFIOITUKgLeN=u2)^Nfb61RN zFXRLxo*Yp(rDYDh%fvX9ep1fB_%NkMKaa}KZk69wKN?=$VR)i&3g9OP-^!5Tyl(F& zJRDd|(04EPZtu~Lq4Yk0i559l)3Tgu+C!FyddQ@u_UMnngX?cfuVtMAb2;X9q*q7U zm*#e~Z&Z)Hi!uH?zDGY3Yn`r(luZWcthhDC)Hs|r%w8$vvY@j7Iz#XD$n_7`Zj3y` z#}>$p{u@ZVAIh|qopOrvA39u3n9m@)KNqe8<3Xyi#-r|chUK*!59VQJtiMEn!rhyq4{}*)FA6Hbb;T0KeQbG2T1BN6y{5wki89$BR_a55_qW zILQ=4rVTQfh7+zA4H-;Vv5bJSO33Uj>d}vng3MzP@}c$yQ$EZ~6{n;M=gFz!j8t(> zs+g-T@&U)E3cv}e;&l1~lROH(lm)^tNmp%1F(q>3};fB3mL zRg|P+gFyWaG3Xm_h{4lAH^eMXjXTZEz{^Rg0ujhd700HE3sOa)ij(b=ELg$5A?BD= zaklieQ2xawB8T#VU)7&5{=v6fUM3rSFxFgd`u571Z2-@R0Yb$G&#p-om!yglQ^nP( zVlfg}-k0KYQ^onx-T9|faUmrdX)5$p8;OR0MCbBUu{1UAWM2&>8d1f=V>iT{k{Xw9 zW{_wkH$2f$N*Qs6U0FesCKC2HPXf zG!8F^&gxq*AC&-|CqQT4y)Nn_0+s`azL$Y&F+1CRFTZi?fj&}m7p zT+HoH?{yBF^wPcvng7t0{7H}g0^;hL*_U3Kk9jKp3yo}4&7!hXH~ejczuDCPkLmM= zel@HOG>tj6<7~Tsw83Qe+;cEq{;Egcgm@YKqox0_uD6_KTP$YzXo_5{g}$Yjmp;S+ zeQ&}>1IwB9p2`^^AG4C6Zx8h4evA1yveOMppYcv^a`&Da=Kg21JH;mj?i(NL(MMCc z{{S;Y1LI>YSFhNn4^mmsw+#AbU~cgXoVyYco}~;>-^kA_y%5$na!pkI7=06{9&?Nj zU;)6jJ75>5`MwJC{Mpa8Hwc(AN4`-y6L3S4j`_j_O6MWAcDIa^wT%(dNuSaah0y2P zhxq^pkjBZDbW$tTpVVImlytbucx{9pnxA}@^n7ibR%^T}Ba{c)qA9weZx!b0Vo4tj zr267z%ro(OGu{mwM6mU<;D5$>n18_>r)+oc3HPtsto}r4c>hcP;Ks;6YFo;nqrL~< z;-Y@ezAd%xRiY>RZ8k@sb3r;N|8_v<4y{*TMCm#f@v{uc|A*j*e{O z2z7QA?8uZ8F zMQv5eg=jDFy+7Ihet)2y{jFs*V*3kqa@@Dh1ong*QUi9?UZ~k}e!^ z`0WB$j&JH=+DjfZUTO!o1Khufi4dmqGjla1f5D|l|Kk77Ul9HZ;LnBc0)9G3h1b7F zLVI_RF`9=HmGM=Otv?yxfD-ALC)=xRK-SC830U7tp3BS}{Xy1LK* zpRb?D^k0JU{DgtuSEG7E-ycmt{hLDL>}{A_8Q5-NuExBF_zm0n(L9K$aUXrlv%GLyZX^$dS*)dx9yvE)ZV%#=qkLVSHI6^ z(Smt~M(oqXt$xoQmu?1WJoLxHTJI(2EOHPQ84olr~+ddqj(jmWP z{93O*jf`Yx6U$t~s0j6x(ouRP=3MZ7-SGTa1hAZ|0b@(=LM zTbHCH*95Naqkg?vaB2lt3oei9bzggJw700#eG?GcROf&e7wsIp1+PMQ_@3`sRHp{$ zC=OFG7dsBne#)6NUJHj8(y3w;0jcjMu-jR&>c zd!Zxit6u$FTOFo&EU;Aq9aKkV{2A^4H`wz)=^0oaEd8`LD?-1TMR{2cT^o1z>ZjwJ zj&%Iif(wDWmxw-_TlyxvJv985ItbAn@K=CuBR|#e4W2LG)ZeBR=k>kF=OV-;a# zRE{rW`Jnl}tV?7EVlAmM{eQvS6!aK2A$7X$xKyAXU3tt~Uh#@!$PY1MhvTq4G2&_G z5g){e-Oj*!F=CD8`Dcu%ba`Hn5j$PkAI6A(xIM4Lh)3NqFG1ezei6(dH@?BPVbF^( z#8;lbe-$IPMkm$9h_x}^pJGH!jQ7pkL|LEaS}@#COlIIOwiQH1-b=Gh8A+G7atGne}{ zkNC><0?yY)x&P@A4@O~6X?s)xPInLT;$w_I5AtsD2z`*Z!6R<=V6k1rdy*hOFE~6~ zyy7*->S!T8jqyC~6<^295n{d93*$cRJyglg6K^Bjw>9^kc=5j0gSfsM<^C~V?1}i*%)Q00_belBygD!ES%lo`b+~t}@UMQ|di8s*Lzv1%i zixzLYHX+32QSQ!YQ66DqO}qT8lx zj}1%wcDlALA-Z+CrY9tRHC@{`+_Pi4wrj+M4KuZC6GuKdQ+wJUdsn9Rsvox;PmWBu z_GoSA$mr!qYx>Ceo=olQ17h#Y)P6bu^ADemO6bVcRwPAtWoora@xNwjl?TS&o~b=_ zAdGO!=!9P~wYt&KKW1vXN5}ssQ`BA{Q@w9wYOhT7LglupL(qv%9r4i&?Z>IXni<;aBjZuPjvVpnbZyI# z!H=hFdydS&midek?@rfV$_TzaUE6rn#7)z+x28>eV7m67X^$XARWsfHm?`d?DQkdm z{2gVZB~6yE!Fgj*=r5b2%8TxZS>YW0j#K=@Nes%L*?moe)A6kH#Kdh!Yeg$i7WX(3 zo;+GBbFRkuYAyQb>Doiu@SW4OCN1&h>Dn51^rmUrogNR2zkKiz}WiCAfT=_U_bqLfm!a=%1!(n~n@#J5Bri$l*f#eB|g4rfHQK!B39THfO{N zu`MIEVVd?y#)ucEXOnZiiQq0y;l>|E0#G9*m9V7%yGcx!^8)U z^cM~jZ#kp84iz6bQ>kf~@bjUf!4>n*DWcImWBXyEDJt&g!^ORW;@0APP-5d0G`zPV zZ}-I{J~Tz#;=K+kZySsOfSq#8*QTpHCAVLvNs%4NKZEMXVhb_wW?)!?5_CG*OihU6&^6 z5~3^9#Lo%QYtqDH!xLAhiC0Is8`8w`#KfD@#J!2Qy|~4n^nRLn#2@!&n)t#W`*WJO zdt~BmY2xvb7?(eIK+;=j;-v%P-bfRB4~X5CCjK#M-1}3+$D@w!KzK<-FCQjW9JtBx z+F{~_(RG>-Ka4HK`M2XJoxe;>DoYc$PjdfosJMI5mvH^{;OGwy71tgz>h(j#+Czq~ zJ5+QY;#qyDcsOOow}*<|DZ@WKRNOb&-!nx#JK5hgMZ7)PyAy#;_C7L2+?DEWN)zv; zdS6Zxw;bxdK25AXbPM(W`kJ8*kH;PB7?O=FL(xL(EjfqVdog_YVo`E{1jr?h%xX+ca{2)>0_C7IDta2~Hz{EZB zfr;pD6MhVerw4iOn)(Ia~%h-YFF-VTbp zz22?~;vZhnA@4}Mar~Ph2`>gkqtE-V31Wv2^rNvOKb;`%iA#7SC_3Z3A5Rb~hl1WU zbmTuLh_~Vs?g@&G!@P|XMEx+(wrFYqGdKDPij*u{X*4MY1S6 z5cJLiN4}LTULT$C>?E;%jCV`2czO)z^1#R^lEqJfgmo~|Snngr;@a_EnC+JFURY`M zcrT3d@OUq5@YHxOlKj$mFH-o{crOz4;dn3N|K)fuqWkahUc|0rycdzVKG}<~Z%_6j zsQZ$=(ELa;25K^YF5IHI-;NcVUG7b>;#JowWZT<=JdLs9!$DI822X=-jTOrWyYW<5 z`Cw$(*Mk$EiWQxM$83xhyJOs)v0_imw-`vg@AG!YiqC!C?_SiMcIH&oUuP?;u+`rlBRN!_>p7SS8mbm@O|eNpSlONx={{jo=jge+!3AN z-sN(P+~$(CfHV<5)DiUn#)2+~QbHVdv>!hN9l+T!s~lo!ft1r7BhZBi{6^fYi5Hy{ zTAV27N3&bHsE7A@#}GUq{;nhXc};xo7sIEgp2cpKv?YyZe+Y{V^XnqN|+ZGsp4L{}6}E?e-eh)7Mz8 zbw`|_k9|Dk7_;4h@iLa35XU!-~ozUtNpLvI1Wau6B z0=x{FQ125@4vB5@iI<1C*ZailL%a|8#JfY#54H>$hWW*PLnf^8i8Vgz`Pmjv{OWdhyM^v{uZnW4pjW<=s63F4QGyoJo^`nIcH)izvAW$!r+3SD^vE(d zD*AEFF`)uR?f19lAjcTGJ5C?^g~KtIO2BSM+^-Il7WWNK2U)C7SYI>6F>#9L0jJ}r zFCllY(~Z*yoRE0Z>8^8%=bY{>PRC{<`t?&8cnTf#Q-g-A@uJ!5+en8`jK}*iqgQH<$zP&?^rscJ94)2`F+Jf?hvNi{%wKT0 zU!j8Le%Il6Gu)I&V_zEn&)9u{4bkR|if(s0CVVh3&e^!(ANI7vaVp}B*uLoy|8%%N zayZ_##QA?s$G~!^JErb-1fKMW8=OgB4;HJm5s%}hSxdZOu&8#QS{9A6UyX?0qGbV8Ze2p1?=R9$9 zAhCC@XbKGPm@B>sB;GYoJTZ3or*p;YV-s8EidEx=zcg1oI4=IRxnlRY;dl#H_qh0N z+2XbF!|%)%AB~Uyd5(A@dH4-;#Ouk?c(qJ>a{Le3;<^d3Z)J;@Cd5B8SG+qR_U5_b zrit+{%n@rR#=bL0d^9nB)g1BN#00#iWy7TS@3O`7lVab@7JDbf*UuFzgRwWx6|V;4 zAIcUV1Y_TtBW^t?{?TkveNe&!+2Zqq;#;#t`$4g9WQ)fRj{jkfcb1k;pUf6J4j=wtws}#U$o4|sQ&pncp`D8p@_xuM@dpQHj^;})`U3y84sk6NS9$F@ z(SL!s0QU>B-5X2g{*bq=<$nL$J*wAPLS&72#Nk+Uu0Sv0#)EjTI^6F#99u1Q?q0dM{i+qtdn$!c8yTKd!^6H`_ zmG+#%>a_~vhG8s;15LX;THa86-dhTbS-xPnSuW6|+{->zSi$lQezRQlm#nW%VS^z* z$}HE=t=xrM@f9pzJK8MA(G-H?-FN-}vy)SAQ}MoFvBHZPFK4ty_s`5PW4xAeCF9ME z6^wT=-plv^<2uGi7&kK3G8*CjWs!_%Y*`j6X2`$|wp{I${_{FpgzR zVVuS|kMT6d^BAvSoW;0|aXI5HjH?+pFuuZQ&1dU%^HOt9QQS$nxO&0&P7XPZZ>c5#cuJ>XFhYsUO zkGS(xe6ib29)3pLG07vy?n!)Yr0$nq{ZgXZxcgLA!hRP_$I-#PlL+7*mku@ijn{Kw zx`__jc0h*_9@bh+2YbVwJ{6+TOJnw39tkXM1x)#)*tm8Rs!BVqC&_72`_AyBXIrZerZZxRdc) z#(j)YCvo_U6B(y6&SPA}xPiE%6APR4H;_c2CcJxqqrIFWHG<2=Sij7u1=VqD31H{*K7O^jO^ zcQSsiE%6APR4H;_c2Cc{#}O8IFWHG<2=Sij7u1=VqD31H{*K7 zO^jO^cQSs2%slR%vy2|P-$lcYSI;wc z{7_N)h8-U-s*fI6E;KPr7~`6L?PP}pVT@J(v^bD*=^c^g__oCL;Z6($inRLfl@qvwRtsw;;DGA<4&zvSU?!ui^*|fDL5xKPO({4B{+{8fIb*b8wY^EI3QSdosa`}jC{ zP1dD@_LNh;)NYSA@oV~cA%$Gf&`14tw-XB5oaStlq(4WG5 z@k3_$Y0S63tau~bW0-GgP&`jVi+tt_URV6}94>u-OUAQB@mDcV-(@Dg;TOfzMhQA@ zWj?J_U7yT+4f9#uil^t4>3Es>+Fo`2DDyj*FXp`g+nA?4xD;+#KpBjumj$-DsQ64) zg2qP^5|KY7Up-FA%kNVns1)Xn`Fd;}mPZcr1#B2fCmr-YKhk5&pBvweE@Qr!=gUX4 zp0&&y^W(;Mn(M(+{i@*lWpw{febkmvOV zL(j>~8|x7NVLcZb^1L1~ob@ba-dLY7^jyunu}%@q@^>+BtXHgI{r50$tY4(D{3Fbl z@jAwzm~UX-SjWg`{(a_+bq(Visy{Ga9aQB_eybB<55hzZ*}3*Y#mnz%f)6lXe3Rnk z_c6gwV?OPE#rLY_Se$I|>lH7*R|)y0hW@7%FTXtr{(3|H6~)VMNP@qg`C@E*p+kNT z5=hG`6wj7zikBa}1OFcL74Iruey0)qzYIO^DL#O5Mu&#@lAekW6)(S?2!1T{wVx{f z6_%gQd~u87e=75Nt%{f5D+Eu|dZa(FNAdEzgWxNf7u*3-`$Y#0cS*juOUcXc z0D^zpkf#^d;E><*1AcGFAFlWou3z+^AL+@Pqj>p!J;)!ze0#3q<@fNwFJQjmRK?$+ zxE|{?2^he8r!|_F2Px*+q)q%X(fm_!7m-@4P`zGxIHfSA4eAE<_je zX;&*gi}^8_sFU_#UVf_$Zkfzi-=O5>H`l-yFduh^;wJ@^qQ5eqMO*rC$ZwzlS2Lft zUGef;X5i}#J=md0hx{fP@LfaChl-cq7X!bC`LaREFu6P|7>rviiht8^#mnz~L4G3h z1qUcze%lND9OeU)6feKi1^!&-<4#q){N5J$Ynd-ws(AU`Eb!}@FQ&cKIOI33fLoZ) zyIb+{dsX0Dna{dc@$%bK;C1HH@FWWz?eeb>qcMR?=?Xlfc%!{Nn)!C-<@cT7X9@Gw z4N6{qiwXRlhWvYqpQEaQ*u;F+4#mswAVL0P@OIM0na^c=zc zMc;jpMg9u3D->?UTS}hZ7e|M2e^h6YkCVe)Jg#No$1~q<;iod6Jj)#a1>j}A9H+vi z_mk350G`Ir)jaM^Q?$62c@N)b8QRt{-)xb8gZV1v4f$5)i!AckJu2f9M;$8;ddD3d ziQon9a|;(pCz$#3SfsU`h zlilhp@%e@MP_{X}9(kLCa%G9;x@A+^nFrRPX2g`-(cu`{E4`jaD!cSp- zyM>?2ys*q$o(7)A6@?{gT*2LuC}H_Hi+nlrK?}c@`Gpp~mieU?ehc$e7QPugwZq0f z1zDU>(w1V%SdM>qUqOW|h4_Pody+TyB^dJcl8+U}K7=}!FUB(|cH^tRftTg~VRb_w z-$w&Kt0Ks+Hsq%$yUF)uK|a#_=LX0-#89!;Wm)-gsbLFE|<#VosZNjYg?| z;C%jj9M8NTRXdDfq&(WB{$T^w3p$v$t}l2;Q~Y7wVsm*O2R@$0QC&(ePpb)f6`s+C zaF7yCiol=8da8XYo|8D3o0+c-DJ|PsSvB*;lUP2W9A0GJ*jL!1vPXO*dG;Iz7ZupY z^B2gEL%IssZiVb;CK^miZ!Jv_;W&i(#o$T4xIhs)kE4oNzF~vnuV(qbFdukQ5zClg zW$3?65%h!-9c#hMA&t81`&(=1`MaW&HSoY8c#2P2oGQ<=*vHp~{w77lvHo`E#U+YA zgXOzfPgAVYBfDw19T!k`2y9h?yggN%4xYko`c=iGhxMN~mgSA{x$-7T!P}+l0Ym<2 zC3p?XKgE2Rd>T>otv7CCzWO;Ocp1xoD0#jJgXD24KE-T!zO4~cB_Afpr%$qc%gc%| zPu?QX@N!}8mh&$?0(k1F$t4<*k+VbBSl?A9m|&e&U}CyPE1G0 zV=43E9(8Tl;Y#MKx!o|z`PJa5{KQRAIhD%=rE;R;MY7^!nSXB*rEB0ZHmKy&5|sQG zEV2kZdG(w7itNuDpl;F<{hU96|}JS8lL>p0y2ekjIK z#=5_Yb!}q#ik}tH&iqy>&lh1Z90gU{xm?-pL3Yb!UTjpv6P%A{f~S11{z=&(kLAmy ze3)WBeS+o10;T^W<_po0QMhSL1X%y&k{_t&K9(1^Dz|kk|9AxXUY2jN)YG^W6>i)F z)luKeK?T55x{A+L#B;1ao#oSBSG*iX;xvc(>L?|6GV3`<@_Z2nWrqC!*WQ64D9qATf|i2uT1_+;eLNf$HHVx#{UrRf(OJaQfE0r>buHPJM^!u7IS%dy0xc zLzqNgC?rfW*kFhfgx4zg;5{@A`tbx28z)c%3M8mF_1UWxZ+}AoR?G3{FB*Q2v@@#1zh6~+=!bGIPvM7DeEn5|+x-4{ z;1KnR-?x}P0XwH%Bki;LK6&v#neldCV3C5G<{F2fieu9isJHzi=ikGlilzztl zI0#1iOV^RD?&szNKOB<-&rp2z@8#Wh%f*=lPW4@<5A5J)^U)Js$!GmjLTG$)1>=H+ z|DRI&!`Dd()h+Ja1f238%1Qn=DPO%p@!V?#|CVO4*MO zD)c5HcYa3kb+v<}v^cZ?_ywn5l!#}1Pe%TiY>{@beO|^tAJ_cXRNnsKljOyJkde=K zlz#o=Lbs;<_|JylE%>ck{>O|-J3HT$H=3S&h2rh)QX`9(c)_@oGuM&~&eME$D!z7) z0Q(fbS@Au(4-V=U|LTe#`k>tJRlKjb^Q%&x$1DCy#oIbA48Ke9Ttf&Af4|~;-YK}U zp>vPX-z@d|jOPC@jO$J4@}m<{-_oxLX!^z^;~_%*v!wKgVtQIx@irZAg5Qg@U-l~Q z#OPn9_|X{t2FCSeq09S#^Z1>Xdc92Th>Iq*za9``lk=mDhceJVXO!MKOA2WH;wygW zun^ifJplZIWH!!6fKz!&FP8F{yjq>2@+5z28Tdyt@Xu!8Ujcp@#@#NBH?w*6A2R5# zc%HP+>XjnTjsE$HmsBnp{}dEI`XS*@!*5|cS+Ac``qeK9rFOgXR>g;YUhfcpY`WMF( z@bBI}-g|m79{79uUgtGZU!BIBvwtkToEHP9{GB&S{^nP_U-9+l3jPeO^EWc`nS6od zbM)th{>PR6^}t!r{|5nVTzrV>l_+#MqV%?(m(PjD^S@R6h|c>PG|fLLKJ>HlUh@lG z{X%(fTj_1=j48fB1@lvMKH8%AYD`bK2{@1UzmxX7OWVJv^oL^fKdbnjn4bL0iXVyb z^T#y*Lot8Xy3((oCGVwgozm|yFOquMzUh+6pGy>P?~np+(Fr35oXS5<0ZaHnUF6>m zr62mR+~2Oar}XU@{fg4B>HKTs>y3tgR5Gyn=q}A?UF!w0&%bl0)A{gH#-W$KUFdAQ zUk#kwe~a|5eJA)g6yI>U;8#jA=Rx2$))eB)1sl`xZHz-dQTxTx-eUL_A~AMrW-Y@v zpT>xW!#Z~Wr*a;AmXzmGrGLgI=`Tn97$)b(fD^s*9qE@Fv>&%A{XvzJe4a4!swzID z^6kZ%|9;Jk@w!H_{ECbK6pOE8_$LrSEA76 zh77!#fxiMc)pz{{DX001-ciu@8ItmRSj+!$;6&d(Tj(tw@9&hp{dy_Phm`(|YbBqf zy6zA0$iJs5-qrxy%as0V#+4{^d9Bg61^m>Ra(!zC{YRAk;GYPciWTQDa4P@m6CTSC zo}=%*=sGFq>T86)Cl}``#n)dczz_Akql_z2=yI*mHwAoxKD$xzq07%=gyN*>x2X8q zy>f5kbid&rj^cM%KH5iBxM(jWYU;Sw#t(geRld*;fEo$D@|@ZNpU-8 zg}-BfG|y4oiLJj(1E+pzUoDJzp|;OUm3~94zCESibEVMR{P?(SQl6u#SD0RRjpEMl zNx^O3!3z{$J0!UA$+e0ftO;)8Xa+cyb6x$_rpLZP>Dy|5ee7aM_in{Es9j9^k?8kZ zis$Z=`(KrdbHC!#$`5BN{wIvkKB#^+ZW z{mUgEtM8=ZPON_mif>SUu<^UgaP41{d$%e+9n1gaM*nB>-ml5Uc{Olu=Le-E*K51| zveK_Ngz$W&|DfV)9}>XY;gi70pJV&2l-&7WOmBCC>s33_$Mr78vA(J20r59AkgTQn z^k*aw+y8Pqa4P@wh_t|8Yx%#V_|YFoy(;?ZOLj_sIe#vIvXOI@;)lL0_Ky}zyjJ=M2;m9)Q#Ip+Xy)5$}{bUv%} zM_w=VU(|=*i=|)I{#AhU6>kHl{Ewa^^!IA}Tsk9o?r)@j7nQyWoaX(ZI|ZNvfC@i3+Qy=3$+5yI2X5qvp={%11q zuWCNIn4b1s#n*pZ@_o7%fldieYv&z|gAeZzLTl%82L1b$euJ(rKU?$vWCr~gl>U(V z87@-#ZvvWI*<>U*y-{B)tDeY5o24xH+BNax4Az|K!A z{o0!axJB_dE54^Gz@*~uwS3giK;MR@-}$gpDDEX5&u|HF_P?pZXgv8$#oKR@^eb|4 z#uYF9jsR+gIxk{ei9(m2(jTS`X!!BBNpbx?1mgz8OFvcj5%5F&*YKr0cD|pr=fyuk zc=Fc(PVIJBD{69M-%BK)L*J72w|2Xo@nk-4RQl4p<(bv@-5K=vDgElph5k*NdgY!k?e-d}hSOGjUB~I$&yxJ^1y23wd{gde|2h4BmvQ+! z{QqBM;16m(cHZSx%AZeuspP*$&pk9bb2V@(=bFxs5aayYrg&~l%1_^2rr%3Fp+B-y za9hv1bx(T#-VU7dpMG7G{=9i9&(OV;E50El7w1yNHz^XAC%xql41U=WBoYRms2fr$Vn6XB9Zn zADt3<^Vi*?<9GGDLVxb*a^0)Ao!`mdmc*mm72or6p|kORCvcJvaXtU-O7E!u++_GY zj7uKj|Gy5L%6aGw(Q>XYr1RU8q2Q&-nX5AJRmL$NeMLs}^X1~)sW^SVn0_jzop&pK z^e3e}e|(la|D56n)h>RFz|NNyKl~HYA)iwGdx{VJh5*L@XD(?!eqVC9P}_eMIPuAm zR|^jFpMURAd~H+;g687ihc%z|H%mDmula0&{Y?3+s=)i8(%-1~^m_y_`MIF@9!JXm zS4w|7<9ZXikbO@1AC0Y>+^zU}4F8zo(;De9s^$DbM*e@9f&YW%Go<>4wdXmvYJLBY zl>a-@9M01fxAR~5TZpK~nBoV&A(b-wXp`b=+Q6p&%qc$I68fFm7Inqf|3U!sZ>%t` zM4`(MET8uZsNLY4xmU`6NEZg_Jbd~+Me#M|x6dg4Y{QjdRn2kkoS=M?zbVku_!@ef zG>plC|58t1V{IX|uV(RT@sX6N6*jQl?goZ4qq*-Fw^d`;zss+G=vQ~cl;q@w2Ec#Gmk zJ}Vi#R@>n{iqkio>35A>oR2Gh^i$D%e&6!Pm3}&moIh9G&T;1N7vuSZimxgCxtt1Z zfcH^5OzXh9N%6~oQ+pnHoaB72mh){)uSB8CCo}LbYW`~)m-SRl^L5RCkFH-IQ2aSC zOw#!9TE;OhRB^EJxI2UX14_THcG=@K|K9*k^*XHU8hSi|^P?TD?{`I>e@^LN0i4=r zgSOif6kk-nwezqg70$^~`f12N`fZep^ETi^`gizpuhO?aE~rl1&Sw>0(|oLdzoK|f z{q1*YX5UeKNbO~vHgTYXmR~czTInxfJX{j_`&q!LeM&EsH~zl%`-0-rrwRTf#aA`| zlFnb3X_l{5`aR=9Z{zXpJ;}cv^Pl_)aBAnn8i$7Oi}LS#jBBQ$%b9(lFFi&IqGH9l zjPVem{+U<$!>-VORPoK0&z$`FuWb<}2MdWcmPoQ<(exu+U#97w27! z>rLqLea+|KgmjeoE1nJkLivq-ptv(C_}%BrOI`w;%6W81@*mcA zxJ~i)zer6dG@lEhn9+L=>bn1omi1|hAH7KEZ&Lak#`+P$2w#IKi zpac8&6yNhIDW~a0SKX%V|0T)i^P0a4oa!}nhR|=)4r?>L5``|W%fQz(|3e><3_hXl za}V$fLh0?#F94_aPHWrM*cgLevJ-*!tb#1b6N3KUB5PcU>P{K^FK*Rc4)u< zlhSWc`Mg2jd+tw4IS+kK8U~-c;@_pfDWAg{$NqYyuNwU`&tl}mioaZO`_99;kCO*) zRebgPa(}ndzn^iv30?jNaC+~Nf0Xi@{`p1VY5jZtE2R8K)LyoE=W@jl>HKwzzGK?( zj|l)a=HCo(dT(w<_|y1gF9rg)r}`^Q?mYn9WQAtIm$O!-JZ+tDidvwH84pv+-_Hl0 zRwYeQxQ=(-ww;?cZu7iuf8Hy4l#$07d7+HF-Lt)Ilb%;=)ta?lBA1Z_GL}Hb6No#S zfbwn<$|s@Z`|>GyO(YXe#;aRtH5#oZH&&(7>-PE!3&Rzs>UaFbTDRwSyk5hr)LTuz z>v&$ZY}8RC?mur~B=0S_IP_L7%W$oW;&-||uTrkpwHS+j zz3=Vo*D8CtG*Y&14ImXxZv6x8&>g3shg@P5;`)uVUU5fpE&^Oz+_7;RRfJ#P^!Ix` zzu9edJYs19%hhW1&JEvIDxo`K=t06p9~BC+sn(W5cU{!Y58wN}&|Rgy61wZO+F{a0 zx$Vu@n&pnrP!CB#=|QF6?X?;soI34ExacNxR$uH+2PC?3CL} z9&uy8=X<@Sj$f|I6T=o-o&Dub)!ElCH+!|)e6Q2m@9i7)a$d(@0K+#ce&U&%`E2YM z>BhY5bd!;E*IjqxwITlHZeR8*7(~56dj$a4r|Kt)uT7$ooX(v#F%i|G%#{YK^kVsjlM@i#U6ax*<$xwF_PA4q@J z@!NhmqlEGdQna!+JuwT(?8nVkC!;2vM%O$3(__4uf>j$ z#3Dp!cFWt-W0bdAD`)gJJwa8vdoe*Rq!(*Ia+}|2rPFj<3q53$(aZMq7$u~cAU$iI zBeHwBT(8a7(o^Cq0F8`sTxm6zJvXC^_+dWdVY}6;XQt$bSt%FlAbjPSH zgtXTs)errZQR4yYiYj4xJ?p66?%l3i^((Ea@AZ2NlclYd!g#&a+6&<*~PuqoVe*c8x+TTt?PXyjJASaQdu7`@hBzv)%W zy>flWl;&cr(J!ZQaw0U}Y+IvsrweDo(wY^r`+Z%nUy!YX*?(yHNm5sB73mYiC z;ADn2P^j6w6XxicC*<^i^C?>P-NQ@;r z{;gnFm?F(;4V>Y39|8-O_&V$bVFb_2ve8()@ zmb4L=pGpkMk)*{uLaedP9p7Hq?#&cl?9FbT**LfP`dkfgDqfh_3opTKWE}6l?xqwH zcR5UQTmsG`Z?7-hv~gQJefGm`<7jLsIGe^GC0rMh@wQ~0lg%)i<2pz4O>G8uIg#J! zcvi)8wL!(>tc}gg2`E{xF_Mtm@?*k>Qa3Y)yG!M^uN4iCfLp}@U3M7F!*AkuYBe6(Q*57Lu=}N@(gKT#hgzlvCcHX44r;k#%Fr^QmJn6k<8FSw z+^tnG-a57BVzGkR#jPx1&g*r`wO+Sap&6_KnT1ha+G-PHEN5sMzCKw6#{+x=^d(|4L%qOeB4L6Etx^keHBcgByBYcLm5sHRjeFa@PQUrkO7`$dj*}2z54B1E!wNoV_W1B~Q&tJwe99=wCs!fd zd~%?=`J@zOHO0n*3d$S=lPvZvjxXcrQE}3QR79#Fm!YMW^G^J_u9Qq(VQak z4tky60m*39UnuwMJr70%mTJYFHP-RFey>o1|3ZtW3MZ(9T5JKd#11#Nh$VEn-<*dg zz?BSE9oWl=p5X@7XS9Nd#aj4lyS*yh>(Qcf`oa1py<^~93FIC58p!juC3xj>4Qrw! z>gH|EufPK$X(zpzc^EprU!3qguKsLqv|X-sisYzk6sV!RSyaE$bGyCb?1%?wwC;Di z{0e##pwOGpI%H%b_t{>(TWoA=jN}X38sqs<{68`^LI1n>FF!UuF*-Ri?vCNcC1f-| zH99puk)I-j=a|jfC48?iZGJf4elVK538Q2PaPaTXv%;WChd z^=fe5a}10)Iq0^bD+AFbBMfP9B{+vV1a1iuc5pvTAf5v2MldH*dnbsiFh=3mRN8%S zsf>V=Wm~OMV7Y4mttWUsdCQ}6TG+KwgNq$LAgKhcqq4+tN2P4F1?|`rCMaxiQ_C8Z zsRKdYC`-hJ{mf)bCq)>>u*J~9q&L}lFWQqUY0d|E{*xrJqz(>V>;+-Y< z#E{!YFQ4Z=m5V#(Yg!ZUIZQDG={XB#^uJ}!n zoy-XfjhU^D|7;a*mAs2)QPHiM3aNsHMx(^tBwWLin>t+DQXB7NS!f)SHspGtksKNr zZSFX`YDo@3p(51?stktFOgto}09*f+^5`ErsJ`q$C)-_ppM5Cy9R<868 z2rEz4T3w!8Nga#G9x5Fa)mBGQRiQb@g*LQ6XbTH%L@#;8olqO9w5Hv~mQf}Mt#Ct| zm*Z_((7tcKdu{ItOElgfSjj&7sc}Frw$A(GA za08vF$G=3&6bNV(<~D&6mMEbb1f29_hmnretx(e=yqB2w=3$sgA9Bq)9?T44;BX%g z3<{V)ULFyZ;kj+TU#nMX>5{?)5kK69%1H+4fwDJ`u-GPy(q3gLtcU5iW37!F8VpCV zDqed@AF;9Jp{aY`|jPg3f%_3IjHa|M9FcRv1%ogOgMxwS-gOdcbujHQuW&Y@u_UZy+WG!+myErJPBsHNSB4q+ z9Ui%o7=~Sn=h^EGmaS)IVXPNsw|d1JNLt6mHHU8N*nrq*;y#TnNb1-e88XF5ROtG< zS?V2k6!H5AqpBk6n}VteHHwZvD~87pnz>kmEh#E_EuyUB4kN*4vl^M0wF7W)jLGZ? z^;~C9$hbv#1$btOTRYOpB+*z{()&+#LSelO?6}yZh?mifS%t&xwE-W7YVNegdN_@x z*z8Wti#VPLV~2cSkl<6BP0v=NX)L=V~+u@P!`xE*se zOtMOkLb7INYcq4P&YZ$|K@BiV8tWq(k;3ILb;F6;tbn!QUux5IO<{si+spM{4PM6b zJc8uXGO)-(p8Q4DA?^+)>vF9kyJosAn3&Bfe7%i+6GI~{m&7VPjx^yt3?AG%(9WW1 zMCOG9nkkxGW=)M}RQJsU)ugiwn@Ao9vJUJ|J2&7fJJp#^jdetEM72xeKj@C4zy>bY zDBHo|Xwm`hOQub?*Rz?9o}}~K39uZSnJjiDRk4NF!;}%7D>~TZBXY(bc>6|p`hg~0 zp6}{Z*Yy0xd}w;H%`hS~2AUmS(CnL^h;{|Q@Eb$SIhX`twr~cv{q8Z`iJ2au0y2UJ z5aFC+bkv&g=G0(#bQb#*zM3vNHHe>B2Ai7-Gc{!}UFry?Sn>>E;*dF4_1m!1Xi}c= zd@oy-S_=ypYR+=E$vnLfBVr^nn8q$enJBjH2F(w=B!^1C2q0oLFOsGW4*k-chv) zF&MG=rQ7f5dL219BjZ`rV>VK_t6HQkh#O00nO1U=5#djK31<(@D1JyA-n$I5v9g54 z84l~Nv|u<3NAYF}*(n#;0L*>7-GzmEpxd03M+D}0)bTKk;13>v2qza|0Sf}I`D3V9 zW!h0_%+59m)k`$}Zj0ix%|VK| zlc*cE{+Rs0F5$%jrbwZMmg-LFk`D%TK$SYWGey#|RMRo4Cxv9t0* zIp%{Im)vNymJ(VN59TIp)%9rcA+KIO4h=Y-Z%-$$y}s_6p*a~Gj!oTMWV0Kaev8*N z5X*}c*gDcC2bA2A(cG|ofaYc)pTe@P0$C0-Xn>-foRyZMND`Jy73EN)&T#uu2=$G%y0WW*BTCD@w z73LS1y0AHe)nQsx;6$YBN6cKrd2$ob-c2q^_DcD`<0^lFqnf!q?zsASuvtQvmLL{V zqRk71iK-86thFM-ad~Fe^-6Hfb`jSboirt54H+PY&C*H|el=|@nARrux%G77WIv{M zEwdE1l*2HxF1D4re|vQ%%4`GOhWMlEYlBMI5j#b60CK1`T)JB3SgbXVt#EZY9~nHF zflco$GNhIN)0Pjdf{Hy3xrEp%UQHOsY69jta#KNZ$K8%unDnFG4h-1AlnQlykjXqv z%@Y(553yAz&lINkX0JHUW}B==hy>(_s)$!naOq~U%i<11xHsWk?Lrf)dXUrH!GobQ zVUAZ!OtqM~Z?jg{ucr>S|73$f4<;7FBNLXWz>82Mt2q9HoOh;z*vdq`yn@Iz)dAr_ zqA@K&X}XlEyTr-vga?q;HRKHg9e`ciJeFHnU4`6&HLNca&kFWXZYSN5w)(KEg}5#p zO$QH#MKs|_B8ZM6drH?XFGmf#Al!)s1e-JBC2xd$IPQc>S=)lmjc`(G0Mro_YMkBz zOB!oL9j^{um9;Tm!^3~$s)v!Ebg&Pc?ywT)(7N4#^C{+$TZ)OivYT8IjD7QjnnRmXx<_eUcNq)w<+>`T_M$F3^G&4E0`sB4j>7u*$DG%`ywc02oMpvM+Li)fB%igpIOiCyF zP_=m@fx=`o;83YH=^7mD5Mp;)wHS;xDzhFTFlb4ff+H=-CKE}sW-Fx^Y3YVWGv}tQ zEzymP1m*xdmmzb&ER@A|pV!MHLU;*+cd1YqDEZRH4KmY072-63Kl$I+gO#r3ZL&z_ z=;CrPcS!h$G?fjJQFYzI+>r;Bl?uXwmHWff!N3$un+?cW5e}*5)))UFsf>K`(12H2 zj)m<&CXb$R!eWf%@Iu%3V?Hq^htJ=5B%$We0A`}_SdmmD9a~|6jxGBGc~AT#vMPwP zHfHH|{SnEOix$aaaN-9E=Wu*Mc@`c49l1>o9pPfYfgTjz^X9g1#wq}B3edgzEzAgT zrh^u; zW#%&s#t``-1}3#}ki=9FPdrQ>u2J%((4r&<6hS+RXeDWpNLdUyn1Y}p*pVxPumd_i zW?`X@w?zfZ0tM(sA!)gU5|oUoBfFOO6mT!g`6D5g;E_2f7N0;w9>wU}IBm)SBr z$&QYiXk+_H6{ms4(j$-zZuzND)CA9(DxRr0dTNCT3W)}EM~SVK_YwV5m`PDG+&q^} ztw50yfp_)O;^(JG#7eo_i!z+NJbUwiRy$OyJ_T{oTgPZlT7(HQ zx`6PVBDN`bjd{_)dIcTK!CE3zt%%SW2!8IDu;EO7A`8oi3?Xj7&J*yTo*5HE z_B30!VBvh&6$0-Gb!hO`!Lle~#4;R^=wmE^kcHn}X;#3;w8JGNttEcYRA&khBQ;#q z5E_gKvNVf>SiXaEF2MM)gD=ASr(%0}NTkFfbhsgualI;VC zR|baV z?tEffDB5BCD!A{PT?j;Mp%-zJ$SzH<+s2wb@tCdq1`&mm_7_y% zu=F^=_Mx#I>oYh#*qg%6J!u52W6K}pi!p)0<-=N$u)f9{=W{qSrwV(G_xe}*9Xc6y zC1D$otaB%569Kxsiv0u#en;wfa6 zCJW9hHNRes&Jwn&EU{2%wlq&~;DMy$K?WYkf#_s+5(*SD)oU=-At%eR_i>@+l;!_# zx%QZrdKR-PUquc%2xj(VQ$C7SA{x|Yz|ux`W*OS1OJUoIm4+ZBo|PCJtsx;|7z^|8 zRhSB{jHwC^FE^06J_uV64@8`&Kw;~*=}1xceV)R7AsHX~%>3w-v*9k`h6pxfj}~DE zCZc^wEA2*w4JQlSGbd~xRrzm~jsSFxQxK|jrO;NWq9Y z+3;6^)X>EBX!F3wCoa>i;3qCGz?`AY2hC7mNa9d8I%Nj_G(LNYW||W@9td*x*nTS* z&U)_haXVDzx5S|e$Id_D?bg>N*ad;nwErPGB`{x;ubbV3s=?C!mE<7@@u(y^;)s(+ zKB2*%uSwgi6VZpcrY2MCC~+O4jNIpNWYSs4wCGu*6ekz=THLE$VbZxB!YZNLGR;5dj*4P2UQJB1ssDTCO+eE;sYk8E8Wk`6E zA*R`EOd5@%RggT=v+;P;*laV3Z>ue^>`6``Su&v{+kk6hbl~M;tSHhwgOyqi8O1A4 z!GA2gN2+4deJr)YDJpibkcokma0~KWQe;)CmctP;C#2`VV?ZYu2EhPndLja9PfgD1 zP9Vf|bXOYlIE0D0(eziZI7h+D_B}-eI?>k@=jdycLTEu>X2QcAEFowVSU_j%NqZ41 zcW}}M*4@LA2G|aXr5nkKNN@@u9ZutTH#T9hRlOOGdxprzmoP_$)q=`i@edyt2!5t5 zj}~8yJ$K**oT9@UqR3G(u}DL)*TNz;SDD_gmK{x=52rAGG@b_+mcL}T8k_TXSB~+m zA>vfxdmPW1ve`?&>*G*#Y06BV44M+YqNVx}`IPjSEDb_H`++7-JT%x(84)-MMKwdp zkQyeZ78GVu#(&5ZBArZ10t*l9HKM1Dq?u4)MeHn<4{>OGA=_IJGFX_f4R)xJeCMem zUwx&`w#uCvYag`~Xl+i))Urpk9jP{(!nUosX8dgidg0$xw(POqDS-&$zB7y9AG<1=ID|t4<83&u- z3D4*3q%uoH+K0{>v=7QA5IN|CnFysum=2V=f9sQPGh>KwY-n^KN|VFMMB8rN(J045 zw-P`PDSAi~H7PjMZD~pX-a{YvK@0_}xwOA$p|o5;lb6CXA2c?0=dgDZE4PWmwpxqf zfjyyJAl@)L7mLnLK4y?=WD#gP4R>sNeaNS^Ji6B0b_Sc6c+~~b`Zz-;xk>_i6Q8jb zZ!=DoXLW>lCCX!=kS=7znpvt|rW%(G7<|!AJCMvDZ2k7DIKmRg_(jJ4$xQ26>MTw9 zC*Zcj$HoD1v zYhf6Yy#d2;lZ6&xAGYxqol<$;e6*aF#Dg)~FekR{=TuY=S@VVIm%i5r=Qd$}j)kA{ zxR;qkzjZ9EKiCP75>~=uIChe0oSk5lg@lK#{RXC+IZZ@xXwW8J&0nDl6w4D2hb=_j zAB6jn_-T53?Ssupj zd(JSgjtt|^;YFNNF-%`TbB5VY7{>RR>5P|Qe+fQX`iSgO6`9x_UWK!2WXlIMAhNGH zGaQ7~DX&N#_BZ82Z%39~(sH9#LHbsYUJQg^gXK$S7;MDx#?Hg@`w?Vw8vdQG*}Yf) zu!{LOy0c^HpO5>; zZ{bO*UvRnG63hZpgm6BNcvoDkfcHMq~66z&h|E<6u43+;TY7qSF z`a(@$<+u0Pbr)#=HI)9XS8#I2u4|f}%1{5S{6gh8i}-``w)`fqlqYu0Jz4JU{g&S9 zzfaR|5;A9r8Wcaf9z0j>DLti$r+*nRD&k}DkAJ1;mG~?CvvL@=j6cHZhcvxi+x9|D zupxH6BbNRkH7I^|wR#FVD8FU(ME7JvT-V(AZjTGHF~ z>J>{VS8F#z?~SEjd!bO+wf;n-lxzI`p8-rmREMpz!I$(KZj|fs`tJ{s_WV%#t2O=A znqG1Nf&T2?u1A4W``P<*nm(uL1BTc1@&5l7aN>|4eM552-6Gc^?N}PS^t1O`K0gI4 z{Qgo)C`z7O-^-~6|5rG3D6Wq+xTUk}gR%72_cUEqu2&CU&3{=sgCB#! z6K3^qYx=gPfA7sIKs15X+tQzd^i= +#include +#include +#endif + +#include "gguf-util.h" +#include "gguf-llama.h" + +#include "ggml.h" +#ifdef GGML_USE_CUBLAS +#include "ggml-cuda.h" +#elif defined(GGML_USE_CLBLAST) +#include "ggml-opencl.h" +#endif + +#ifdef GGML_USE_METAL +#include "ggml-metal.h" +#endif +#ifdef GGML_USE_MPI +#include "ggml-mpi.h" +#endif +#ifdef GGML_USE_K_QUANTS +#ifndef QK_K +#ifdef GGML_QKK_64 +#define QK_K 64 +#else +#define QK_K 256 +#endif +#endif +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +#define LLAMA_USE_SCRATCH +#define LLAMA_MAX_SCRATCH_BUFFERS 16 + +// available llama models +enum e_model { + MODEL_UNKNOWN, + MODEL_3B, + MODEL_7B, + MODEL_13B, + MODEL_30B, + MODEL_65B, + MODEL_70B, +}; + +static const size_t kB = 1024; +static const size_t MB = 1024*1024; + +// computed for n_ctx == 2048 +// TODO: dynamically determine these sizes +// needs modifications in ggml + +typedef void (*offload_func_t)(struct ggml_tensor * tensor); + +void llama_nop(struct ggml_tensor * tensor) { // don't offload by default + (void) tensor; +} + +// +// ggml helpers +// + +static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) { + struct ggml_cplan plan = ggml_graph_plan(graph, n_threads); + + if (plan.work_size > 0) { + buf.resize(plan.work_size); + plan.work_data = buf.data(); + } + + ggml_graph_compute(graph, &plan); +} + +// +// memory sizes (calculated for n_batch == 512) +// + +static const std::map & MEM_REQ_SCRATCH0(int n_ctx) +{ + static std::map k_sizes = { + { MODEL_3B, ((size_t) n_ctx / 16ull + 92ull) * MB }, + { MODEL_7B, ((size_t) n_ctx / 16ull + 100ull) * MB }, + { MODEL_13B, ((size_t) n_ctx / 12ull + 120ull) * MB }, + { MODEL_30B, ((size_t) n_ctx / 9ull + 160ull) * MB }, + { MODEL_65B, ((size_t) n_ctx / 6ull + 256ull) * MB }, // guess + { MODEL_70B, ((size_t) n_ctx / 7ull + 164ull) * MB }, + }; + return k_sizes; +} + +static const std::map & MEM_REQ_SCRATCH1() +{ + static std::map k_sizes = { + { MODEL_3B, 128ull * MB }, + { MODEL_7B, 160ull * MB }, + { MODEL_13B, 192ull * MB }, + { MODEL_30B, 256ull * MB }, + { MODEL_65B, 384ull * MB }, // guess + { MODEL_70B, 304ull * MB }, + }; + return k_sizes; +} + +// used to store the compute graph tensors + non-scratch data +static const std::map & MEM_REQ_EVAL() +{ + static std::map k_sizes = { + { MODEL_3B, 8ull * MB }, + { MODEL_7B, 10ull * MB }, + { MODEL_13B, 12ull * MB }, + { MODEL_30B, 16ull * MB }, + { MODEL_65B, 24ull * MB }, // guess + { MODEL_70B, 24ull * MB }, + }; + return k_sizes; +} + +// amount of VRAM needed per batch size to hold temporary results +// the values for 3b and 65b are not derived from testing but instead chosen conservatively +static const std::map & VRAM_REQ_SCRATCH_BASE() +{ + static std::map k_sizes = { + { MODEL_3B, 512ull * kB }, + { MODEL_7B, 512ull * kB }, + { MODEL_13B, 640ull * kB }, + { MODEL_30B, 768ull * kB }, + { MODEL_65B, 1536ull * kB }, + { MODEL_70B, 1536ull * kB }, // TODO (likely can be reduced) + }; + return k_sizes; +} + +// amount of VRAM needed per batch size and context to hold temporary results +// the values for 3b and 65b are not derived from testing but instead chosen conservatively +static const std::map & VRAM_REQ_SCRATCH_PER_CONTEXT() +{ + static std::map k_sizes = { + { MODEL_3B, 128ull }, + { MODEL_7B, 128ull }, + { MODEL_13B, 160ull }, + { MODEL_30B, 208ull }, + { MODEL_65B, 416ull }, + { MODEL_70B, 416ull }, // TODO (likely can be reduced) + }; + return k_sizes; +} + +// default hparams (LLaMA 7B) +struct llama_hparams { + uint32_t n_vocab = 32000; + uint32_t n_ctx = 512; // this is provided as user input? + uint32_t n_embd = 4096; + uint32_t n_mult = 256; + uint32_t n_head = 32; + uint32_t n_head_kv = 32; + uint32_t n_layer = 32; + uint32_t n_rot = 64; + + // LLaMAv2 + // TODO: load from model data hparams + float f_ffn_mult = 1.0f; + float f_rms_norm_eps = LLAMA_DEFAULT_RMS_EPS; + + float rope_freq_base = 10000.0f; + float rope_freq_scale = 1.0f; + + enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16; + + bool operator!=(const llama_hparams & other) const { + return static_cast(memcmp(this, &other, sizeof(llama_hparams))); // NOLINT + } + + uint32_t n_gqa() const { + return n_head/n_head_kv; + } + + uint32_t n_embd_head() const { + return n_embd/n_head; + } + + uint32_t n_embd_gqa() const { + return n_embd/n_gqa(); + } + + size_t kv_size() const { + size_t result = 2ull; + result *= (size_t) n_embd_gqa(); + result *= (size_t) n_ctx; + result *= (size_t) n_layer; + result *= sizeof(ggml_fp16_t); + return result; + } +}; + +struct llama_layer { + // normalization + struct ggml_tensor * attention_norm; + + // attention + struct ggml_tensor * wq; + struct ggml_tensor * wk; + struct ggml_tensor * wv; + struct ggml_tensor * wo; + + // normalization + struct ggml_tensor * ffn_norm; + + // ff + struct ggml_tensor * w1; + struct ggml_tensor * w2; + struct ggml_tensor * w3; +}; + +struct llama_kv_cache { + struct ggml_tensor * k = NULL; + struct ggml_tensor * v = NULL; + + struct ggml_context * ctx = NULL; + + gguf_ctx_buffer buf; + + int n; // number of tokens currently in the cache + + ~llama_kv_cache() { + if (ctx) { + ggml_free(ctx); + } + +#ifdef GGML_USE_CUBLAS + ggml_cuda_free_data(k); + ggml_cuda_free_data(v); +#endif // GGML_USE_CUBLAS + } +}; + +struct llama_vocab { + using id = int32_t; + using token = std::string; + + struct token_score { + token tok; + float score; + }; + + std::unordered_map token_to_id; + std::vector id_to_token; +}; + +struct llama_model { + e_model type = MODEL_UNKNOWN; + + llama_hparams hparams; + + struct ggml_tensor * tok_embeddings; + + struct ggml_tensor * norm; + struct ggml_tensor * output; + + std::vector layers; + int n_gpu_layers; + + // context + struct ggml_context * ctx = NULL; + + // the model memory buffer + gguf_ctx_buffer buf; + + // model memory mapped file + std::unique_ptr mapping; + + // objects representing data potentially being locked in memory + gguf_mlock mlock_buf; + gguf_mlock mlock_mmap; + + // for quantize-stats only + std::vector> tensors_by_name; + + int64_t t_load_us = 0; + int64_t t_start_us = 0; + + llama_vocab vocab; + + ~llama_model() { + if (ctx) { + ggml_free(ctx); + } + +#ifdef GGML_USE_CUBLAS + for (size_t i = 0; i < tensors_by_name.size(); ++i) { + ggml_cuda_free_data(tensors_by_name[i].second); + } + ggml_cuda_free_scratch(); +#elif defined(GGML_USE_CLBLAST) + for (size_t i = 0; i < tensors_by_name.size(); ++i) { + ggml_cl_free_data(tensors_by_name[i].second); + } +#endif + } +}; + +struct llama_context { + llama_context(const llama_model & model) : model(model), t_load_us(model.t_load_us), t_start_us(model.t_start_us) {} +#ifdef GGML_USE_METAL + ~llama_context() { + if (ctx_metal) { + ggml_metal_free(ctx_metal); + } + } +#endif + std::mt19937 rng; + + bool has_evaluated_once = false; + + int64_t t_sample_us = 0; + int64_t t_eval_us = 0; + int64_t t_p_eval_us = 0; + + int32_t n_sample = 0; // number of tokens sampled + int32_t n_eval = 0; // number of eval calls + int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1) + + const llama_model & model; + + bool model_owner = false; + + int64_t t_load_us; + int64_t t_start_us; + + // key + value cache for the self attention + struct llama_kv_cache kv_self; + + size_t mem_per_token = 0; + + // decode output (2-dimensional array: [n_tokens][n_vocab]) + std::vector logits; + bool logits_all = false; + + // input embedding (1-dimensional array: [n_embd]) + std::vector embedding; + + // reusable buffer for `struct ggml_graph_plan.work_data` + std::vector work_buffer; + + // memory buffers used to evaluate the model + // TODO: move in llama_state + gguf_ctx_buffer buf_compute; + gguf_ctx_buffer buf_scratch[LLAMA_MAX_SCRATCH_BUFFERS]; + +#ifdef GGML_USE_METAL + ggml_metal_context * ctx_metal = NULL; +#endif + +#ifdef GGML_USE_MPI + ggml_mpi_context * ctx_mpi = NULL; +#endif + + int buf_last = 0; + size_t buf_max_size[LLAMA_MAX_SCRATCH_BUFFERS] = { 0 }; + + void use_buf(struct ggml_context * ctx, int i) { +#if defined(LLAMA_USE_SCRATCH) + size_t last_size = 0; + + if (i == -1) { + last_size = ggml_set_scratch(ctx, { 0, 0, nullptr, }); + } else { + auto & buf = buf_scratch[i]; + last_size = ggml_set_scratch(ctx, { 0, buf.size, buf.addr, }); + } + + if (buf_last >= 0) { + buf_max_size[buf_last] = std::max(buf_max_size[buf_last], last_size); + } + + buf_last = i; +#else + (void) i; + (void) ctx; +#endif + } + + size_t get_buf_max_mem(int i) const { +#if defined(LLAMA_USE_SCRATCH) + return buf_max_size[i]; +#else + (void) i; + return 0; +#endif + } +}; + +template +static T checked_mul(T a, T b) { + T ret = a * b; + if (a != 0 && ret / a != b) { + throw std::runtime_error(format("overflow multiplying %llu * %llu", + (unsigned long long) a, (unsigned long long) b)); + } + return ret; +} + +static size_t checked_div(size_t a, size_t b) { + if (b == 0 || a % b != 0) { + throw std::runtime_error(format("error dividing %zu / %zu", a, b)); + } + return a / b; +} + +static std::string llama_format_tensor_shape(const std::vector & ne) { + char buf[256]; + snprintf(buf, sizeof(buf), "%5u", ne.at(0)); + for (size_t i = 1; i < ne.size(); i++) { + snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), " x %5u", ne.at(i)); + } + return buf; +} + +static size_t llama_calc_tensor_size(const std::vector & ne, enum ggml_type type) { + size_t size = ggml_type_size(type); + for (uint32_t dim : ne) { + size = checked_mul(size, dim); + } + return size / ggml_blck_size(type); +} + +struct llama_load_tensor { + std::string name; + enum ggml_type type = GGML_TYPE_F32; + std::vector ne; + size_t file_off; + size_t size; + struct ggml_tensor * ggml_tensor = NULL; + uint8_t * data; +}; + +struct llama_load_tensors_map { + // tensors is kept in a separate vector to preserve file order + std::vector tensors; + std::unordered_map name_to_idx; +}; + +enum gguf_file_version { + gguf_file_VERSION_GGML, + gguf_file_VERSION_GGMF_V1, // added version field and scores in vocab + gguf_file_VERSION_GGJT_V1, // added padding + gguf_file_VERSION_GGJT_V2, // changed quantization format + gguf_file_VERSION_GGJT_V3, // changed Q4 and Q8 quantization format +}; + +struct gguf_file_loader { + gguf_file file; + gguf_context * gguf_ctx; + gguf_file_version file_version; + llama_hparams hparams; + llama_vocab vocab; +struct ggml_context * ctx_data = NULL; + + gguf_file_loader(const char * fname, llama_load_tensors_map & tensors_map) + : file(fname, "rb") { + fprintf(stderr, "llama.cpp: loading model from %s\n", fname); + + struct gguf_init_params params = { + /*.no_alloc = */ true, + /*.ctx = */ &ctx_data, + }; + + gguf_ctx = gguf_init_from_file(fname, params); + + read_tensor_metadata(tensors_map); + } + + uint32_t read_u32(const char * key) { + int i = gguf_find_key(gguf_ctx, key); + if (i == -1) { + throw std::runtime_error(format("cannot find param with key %s\n", key)); + } + + return gguf_get_val_u32(gguf_ctx, i); + } + + int read_n_vocab() { + int i = gguf_find_key(gguf_ctx, "tokenizer.ggml.tokens"); + if (i == -1) { + throw std::runtime_error("cannot find token list in GGUF file\n"); + } + + return gguf_get_arr_n(gguf_ctx, i); + } + + void read_hparams() { + + // TODO make keysconstants in header + // TODO: read all hparams from file + hparams.n_vocab = read_n_vocab(); + hparams.n_embd = read_u32("llama.embedding_length"); + //hparams.n_mult = file.read_u32(); + hparams.n_head = read_u32("llama.attention.head_count"); + hparams.n_layer = read_u32("llama.layer_count"); + //hparams.n_rot = file.read_u32(); + //hparams.ftype = (enum llama_ftype) file.read_u32(); + + // LLaMAv2 + hparams.n_head_kv = read_u32("llama.attention.head_count_kv"); + } + + void read_vocab() { + vocab.id_to_token.resize(hparams.n_vocab); + int token_idx = gguf_find_key(gguf_ctx, "tokenizer.ggml.tokens"); + if (token_idx == -1) { + throw std::runtime_error("cannot find token list in GGUF file\n"); + } + + int score_idx = gguf_find_key(gguf_ctx, "tokenizer.ggml.scores"); + if (score_idx == -1) { + throw std::runtime_error("cannot find token scores list in GGUF file\n"); + } + + for (uint32_t i = 0; i < hparams.n_vocab; i++) { + + std::string word = gguf_get_arr_str(gguf_ctx, token_idx, i); + + vocab.token_to_id[word] = i; + + auto & tok_score = vocab.id_to_token[i]; + tok_score.tok = std::move(word); + tok_score.score = gguf_get_arr_f32(gguf_ctx, score_idx, i); + } + } + + void read_tensor_metadata(llama_load_tensors_map & tensors_map) { + const int n_tensors = gguf_get_n_tensors(gguf_ctx); + + for (int i = 0; i < n_tensors; ++i) { + llama_load_tensor tensor; + const char * name = gguf_get_tensor_name(gguf_ctx, i); + + struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); + uint32_t n_dims = cur->n_dims; + tensor.type = cur->type; + tensor.ne.resize(n_dims); + memcpy(tensor.ne.data(), &cur->ne[0], sizeof(tensor.ne[0]) * n_dims); + if (n_dims < 1 || n_dims > 2) { + throw std::runtime_error(format("llama.cpp: tensor '%s' should not be %u-dimensional", name, n_dims)); + } + switch (tensor.type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + break; + default: { + throw std::runtime_error(format("unrecognized tensor type %u\n", tensor.type)); + } + } + + + tensor.file_off = gguf_get_data_offset(gguf_ctx) + gguf_get_tensor_offset(gguf_ctx, i); + + tensor.name = name; + tensor.size = llama_calc_tensor_size(tensor.ne, tensor.type); + + tensors_map.tensors.push_back(tensor); + tensors_map.name_to_idx[name] = tensors_map.tensors.size() - 1; + } + } +}; + +struct gguf_file_saver { + gguf_file file; + gguf_file_loader * any_file_loader; + gguf_file_saver(const char * fname, gguf_file_loader * any_file_loader, enum llama_ftype new_ftype) + : file(fname, "wb"), any_file_loader(any_file_loader) { + fprintf(stderr, "llama.cpp: saving model to %s\n", fname); + write_magic(); + write_hparams(new_ftype); + write_vocab(); + } + void write_magic() { + } + void write_hparams(enum llama_ftype new_ftype) { + const llama_hparams & hparams = any_file_loader->hparams; + GGML_UNUSED(hparams); + GGML_UNUSED(new_ftype); + } + void write_vocab() { + uint32_t n_vocab = any_file_loader->hparams.n_vocab; + GGML_UNUSED(n_vocab); + } + void write_tensor(llama_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { + switch (new_type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + break; + default: GGML_ASSERT(false); + } + + } +}; + +struct llama_model_loader { + std::unique_ptr file_loader; + llama_load_tensors_map tensors_map; + bool use_mmap; + size_t num_ggml_tensors_created = 0; + struct ggml_context * ggml_ctx = NULL; + std::unique_ptr mapping; + + llama_model_loader(const std::string & fname_base, bool use_mmap) { + file_loader = std::unique_ptr(new gguf_file_loader(fname_base.c_str(), tensors_map)); + if (!gguf_mmap::SUPPORTED) { + use_mmap = false; + } + this->use_mmap = use_mmap; + } + + void calc_sizes(size_t * ctx_size_p, size_t * mmapped_size_p) const { + *ctx_size_p = *mmapped_size_p = 0; + for (const llama_load_tensor & lt : tensors_map.tensors) { + *ctx_size_p += sizeof(struct ggml_tensor) + GGML_OBJECT_SIZE; + *(use_mmap ? mmapped_size_p : ctx_size_p) += lt.size + 16; + } + } + + struct ggml_tensor * get_tensor(const std::string & name, const std::vector & ne, ggml_backend backend) { + auto it = tensors_map.name_to_idx.find(name); + if (it == tensors_map.name_to_idx.end()) { + throw std::runtime_error(std::runtime_error(format("llama.cpp: tensor '%s' is missing from model", name.c_str()))); + } + llama_load_tensor & lt = tensors_map.tensors.at(it->second); + if (lt.ne != ne) { + throw std::runtime_error(format("llama.cpp: tensor '%s' has wrong shape; expected %s, got %s", + name.c_str(), llama_format_tensor_shape(ne).c_str(), llama_format_tensor_shape(lt.ne).c_str())); + } + + return get_tensor_for(lt, backend); + } + + struct ggml_tensor * get_tensor_for(llama_load_tensor & lt, ggml_backend backend) { + struct ggml_tensor * tensor; + if (backend != GGML_BACKEND_CPU) { + ggml_set_no_alloc(ggml_ctx, true); + } + if (lt.ne.size() == 2) { + tensor = ggml_new_tensor_2d(ggml_ctx, lt.type, lt.ne.at(0), lt.ne.at(1)); + } else { + GGML_ASSERT(lt.ne.size() == 1); + tensor = ggml_new_tensor_1d(ggml_ctx, lt.type, lt.ne.at(0)); + } + ggml_set_name(tensor, lt.name.c_str()); + GGML_ASSERT(lt.ggml_tensor == NULL); // if this fails, we called get_tensor twice on the same tensor + + if (backend != GGML_BACKEND_CPU) { + ggml_set_no_alloc(ggml_ctx, use_mmap); + } + tensor->backend = backend; + lt.ggml_tensor = tensor; + num_ggml_tensors_created++; + return tensor; + } + + void done_getting_tensors() const { + if (num_ggml_tensors_created != tensors_map.tensors.size()) { + throw std::runtime_error(std::string("llama.cpp: file contained more tensors than expected")); + } + } + + void load_all_data(llama_progress_callback progress_callback, void * progress_callback_user_data, gguf_mlock * lmlock) { + size_t data_size = 0; + size_t prefetch_size = 0; + size_t lock_size = 0; + for (const llama_load_tensor & lt : tensors_map.tensors) { + data_size += lt.size; + if (lt.ggml_tensor->backend == GGML_BACKEND_CPU) { + prefetch_size += lt.size; + } + } + + if (use_mmap) { + mapping.reset(new gguf_mmap(&file_loader->file, prefetch_size, ggml_is_numa())); + if (lmlock) { + lmlock->init(mapping->addr); + } + } + + size_t done_size = 0; + for (llama_load_tensor & lt : tensors_map.tensors) { + if (progress_callback) { + progress_callback((float) done_size / data_size, progress_callback_user_data); + } + GGML_ASSERT(lt.ggml_tensor); // unused tensors should have been caught by load_data already + lt.data = (uint8_t *) lt.ggml_tensor->data; + + // allocate temp buffer if not using mmap + if (!use_mmap && lt.data == NULL) { + GGML_ASSERT(lt.ggml_tensor->backend != GGML_BACKEND_CPU); + lt.data = (uint8_t*)malloc(ggml_nbytes(lt.ggml_tensor)); + } + + load_data_for(lt); + + switch(lt.ggml_tensor->backend) { + case GGML_BACKEND_CPU: + lt.ggml_tensor->data = lt.data; + if (use_mmap && lmlock) { + lock_size += lt.size; + lmlock->grow_to(lock_size); + } + break; +#if defined(GGML_USE_CUBLAS) + case GGML_BACKEND_GPU: + case GGML_BACKEND_GPU_SPLIT: + ggml_cuda_transform_tensor(lt.data, lt.ggml_tensor); + if (!use_mmap) { + free(lt.data); + } + break; +#elif defined(GGML_USE_CLBLAST) + case GGML_BACKEND_GPU: + ggml_cl_transform_tensor(lt.data, lt.ggml_tensor); + if (!use_mmap) { + free(lt.data); + } + break; +#endif + default: + continue; + } + + done_size += lt.size; + } + } + + void load_data_for(llama_load_tensor & lt) { + if (use_mmap) { + lt.data = (uint8_t *) mapping->addr + lt.file_off; + } else { + gguf_file & file = file_loader->file; + file.seek(lt.file_off, SEEK_SET); + // TODO + //file.read_raw(lt.data, lt.size); + } + + if (0) { + print_checksum(lt); + } + } + + static void print_checksum(llama_load_tensor & lt) { + uint32_t sum = 0; + for (size_t i = 0; i < lt.size; i++) { + uint8_t byte = lt.data[i]; + sum = byte + (sum << 6) + (sum << 16) - sum; // sdbm hash + } + fprintf(stderr, "%s checksum: %#08x (%s, size %zu)\n", lt.name.c_str(), sum, + llama_format_tensor_shape(lt.ne).c_str(), lt.size); + } + +}; + +// +// kv cache +// + +static bool kv_cache_init( + const struct llama_hparams & hparams, + struct llama_kv_cache & cache, + ggml_type wtype, + int n_ctx, + int n_gpu_layers) { + const int n_embd = hparams.n_embd_gqa(); + const int n_layer = hparams.n_layer; + + const int64_t n_mem = n_layer*n_ctx; + const int64_t n_elements = n_embd*n_mem; + + cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB); + cache.n = 0; + + struct ggml_init_params params; + params.mem_size = cache.buf.size; + params.mem_buffer = cache.buf.addr; + params.no_alloc = false; + + cache.ctx = ggml_init(params); + + if (!cache.ctx) { + fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__); + return false; + } + + cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements); + cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements); + ggml_set_name(cache.k, "cache_k"); + ggml_set_name(cache.v, "cache_v"); + + (void) n_gpu_layers; +#ifdef GGML_USE_CUBLAS + if (n_gpu_layers > n_layer + 1) { + ggml_cuda_assign_buffers_no_scratch(cache.v); + } + if (n_gpu_layers > n_layer + 2) { + ggml_cuda_assign_buffers_no_scratch(cache.k); + } +#endif // GGML_USE_CUBLAS + + return true; +} + +struct llama_context_params llama_context_default_params() { + struct llama_context_params result = { + /*.seed =*/ LLAMA_DEFAULT_SEED, + /*.n_ctx =*/ 512, + /*.n_batch =*/ 512, + /*.n_gqa =*/ 1, + /*.rms_norm_eps =*/ LLAMA_DEFAULT_RMS_EPS, + /*.gpu_layers =*/ 0, + /*.main_gpu =*/ 0, + /*.tensor_split =*/ nullptr, + /*.rope_freq_base =*/ 10000.0f, + /*.rope_freq_scale =*/ 1.0f, + /*.progress_callback =*/ nullptr, + /*.progress_callback_user_data =*/ nullptr, + /*.low_vram =*/ false, + /*.f16_kv =*/ true, + /*.logits_all =*/ false, + /*.vocab_only =*/ false, + /*.use_mmap =*/ true, + /*.use_mlock =*/ false, + /*.embedding =*/ false, + }; + + return result; +} + +struct llama_model_quantize_params llama_model_quantize_default_params() { + struct llama_model_quantize_params result = { + /*.nthread =*/ 0, + /*.ftype =*/ LLAMA_FTYPE_MOSTLY_Q5_1, + /*.allow_requantize =*/ false, + /*.quantize_output_tensor =*/ true, + }; + + return result; +} + +int llama_max_devices() { + return LLAMA_MAX_DEVICES; +} + +bool llama_mmap_supported() { + return gguf_mmap::SUPPORTED; +} + +bool llama_mlock_supported() { + return gguf_mlock::SUPPORTED; +} + +void llama_backend_init(bool numa) { + ggml_time_init(); + + // needed to initialize f16 tables + { + struct ggml_init_params params = { 0, NULL, false }; + struct ggml_context * ctx = ggml_init(params); + ggml_free(ctx); + } + + if (numa) { + ggml_numa_init(); + } + +#ifdef GGML_USE_MPI + ggml_mpi_backend_init(); +#endif +} + +void llama_backend_free() { +#ifdef GGML_USE_MPI + ggml_mpi_backend_free(); +#endif +} + +int64_t llama_time_us() { + return ggml_time_us(); +} + +// +// model loading +// + +static const char *gguf_file_version_name(gguf_file_version version) { + switch (version) { + case gguf_file_VERSION_GGML: return "'ggml' (old version with low tokenizer quality and no mmap support)"; + case gguf_file_VERSION_GGMF_V1: return "ggmf v1 (old version with no mmap support)"; + case gguf_file_VERSION_GGJT_V1: return "ggjt v1 (pre #1405)"; + case gguf_file_VERSION_GGJT_V2: return "ggjt v2 (pre #1508)"; + case gguf_file_VERSION_GGJT_V3: return "ggjt v3 (latest)"; + } + + return "unknown"; +} + +static const char *llama_ftype_name(enum llama_ftype ftype) { + switch (ftype) { + case LLAMA_FTYPE_ALL_F32: return "all F32"; + case LLAMA_FTYPE_MOSTLY_F16: return "mostly F16"; + case LLAMA_FTYPE_MOSTLY_Q4_0: return "mostly Q4_0"; + case LLAMA_FTYPE_MOSTLY_Q4_1: return "mostly Q4_1"; + case LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16: + return "mostly Q4_1, some F16"; + case LLAMA_FTYPE_MOSTLY_Q5_0: return "mostly Q5_0"; + case LLAMA_FTYPE_MOSTLY_Q5_1: return "mostly Q5_1"; + case LLAMA_FTYPE_MOSTLY_Q8_0: return "mostly Q8_0"; + // K-quants + case LLAMA_FTYPE_MOSTLY_Q2_K: return "mostly Q2_K"; + case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "mostly Q3_K - Small"; + case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "mostly Q3_K - Medium"; + case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "mostly Q3_K - Large"; + case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "mostly Q4_K - Small"; + case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "mostly Q4_K - Medium"; + case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "mostly Q5_K - Small"; + case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "mostly Q5_K - Medium"; + case LLAMA_FTYPE_MOSTLY_Q6_K: return "mostly Q6_K"; + default: return "unknown, may not work"; + } +} + +static const char *llama_model_type_name(e_model type) { + switch (type) { + case MODEL_3B: return "3B"; + case MODEL_7B: return "7B"; + case MODEL_13B: return "13B"; + case MODEL_30B: return "30B"; + case MODEL_65B: return "65B"; + case MODEL_70B: return "70B"; + default: GGML_ASSERT(false); + } +} + +static void llama_model_load_internal( + const std::string & fname, + llama_model & model, + llama_vocab & vocab, + int n_ctx, + int n_batch, + int n_gqa, + float rms_norm_eps, + int n_gpu_layers, + int main_gpu, + const float * tensor_split, + float rope_freq_base, + float rope_freq_scale, + bool low_vram, + ggml_type memory_type, + bool use_mmap, + bool use_mlock, + bool vocab_only, + llama_progress_callback progress_callback, + void * progress_callback_user_data) { + + model.t_start_us = ggml_time_us(); + + std::unique_ptr ml(new llama_model_loader(fname, use_mmap)); + + vocab = std::move(ml->file_loader->vocab); + model.hparams = ml->file_loader->hparams; + model.n_gpu_layers = n_gpu_layers; + gguf_file_version file_version = ml->file_loader->file_version; + + auto & hparams = model.hparams; + + // TODO: read from file + hparams.f_rms_norm_eps = rms_norm_eps; + + { + switch (hparams.n_layer) { + case 26: model.type = e_model::MODEL_3B; break; + case 32: model.type = e_model::MODEL_7B; break; + case 40: model.type = e_model::MODEL_13B; break; + case 60: model.type = e_model::MODEL_30B; break; + case 80: model.type = e_model::MODEL_65B; break; + default: + { + if (hparams.n_layer < 32) { + model.type = e_model::MODEL_7B; + } + } break; + } + + hparams.n_ctx = n_ctx; + + // LLaMAv2 + hparams.n_head_kv = hparams.n_head / n_gqa; + if (model.type == e_model::MODEL_65B && n_gqa == 8) { + fprintf(stderr, "%s: warning: assuming 70B model based on GQA == %d\n", __func__, n_gqa); + model.type = e_model::MODEL_70B; + hparams.f_ffn_mult = 1.3f; // from the params.json of the 70B model + } + + hparams.rope_freq_base = rope_freq_base; + hparams.rope_freq_scale = rope_freq_scale; + } + + // ref: https://github.com/facebookresearch/llama/blob/6c7fe276574e78057f917549435a2554000a876d/llama/model.py#L194-L199 + const uint32_t n_ff_raw = 2*(4*hparams.n_embd)/3; + const uint32_t n_ff_mult = hparams.f_ffn_mult*n_ff_raw; + const uint32_t n_ff = ((n_ff_mult + hparams.n_mult - 1)/hparams.n_mult)*hparams.n_mult; + //const uint32_t n_ff = 28672; + + { + fprintf(stderr, "%s: format = %s\n", __func__, gguf_file_version_name(file_version)); + fprintf(stderr, "%s: n_vocab = %u\n", __func__, hparams.n_vocab); + fprintf(stderr, "%s: n_ctx = %u\n", __func__, hparams.n_ctx); + fprintf(stderr, "%s: n_embd = %u\n", __func__, hparams.n_embd); + fprintf(stderr, "%s: n_mult = %u\n", __func__, hparams.n_mult); + fprintf(stderr, "%s: n_head = %u\n", __func__, hparams.n_head); + fprintf(stderr, "%s: n_head_kv = %u\n", __func__, hparams.n_head_kv); + fprintf(stderr, "%s: n_layer = %u\n", __func__, hparams.n_layer); + fprintf(stderr, "%s: n_rot = %u\n", __func__, hparams.n_rot); // a.k.a. n_embd_head, n_head_dim + fprintf(stderr, "%s: n_gqa = %u\n", __func__, hparams.n_gqa()); + fprintf(stderr, "%s: rnorm_eps = %.1e\n", __func__, hparams.f_rms_norm_eps); + fprintf(stderr, "%s: n_ff = %u\n", __func__, n_ff); + fprintf(stderr, "%s: freq_base = %.1f\n", __func__, hparams.rope_freq_base); + fprintf(stderr, "%s: freq_scale = %g\n", __func__, hparams.rope_freq_scale); + fprintf(stderr, "%s: ftype = %u (%s)\n", __func__, hparams.ftype, llama_ftype_name(hparams.ftype)); + fprintf(stderr, "%s: model size = %s\n", __func__, llama_model_type_name(model.type)); + } + + if (file_version < gguf_file_VERSION_GGJT_V2) { + if (hparams.ftype != LLAMA_FTYPE_ALL_F32 && + hparams.ftype != LLAMA_FTYPE_MOSTLY_F16 && + hparams.ftype != LLAMA_FTYPE_MOSTLY_Q8_0) { + throw std::runtime_error(format("this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1405)")); + } + } + + if (file_version < gguf_file_VERSION_GGJT_V3) { + if (hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || + hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_1 || + hparams.ftype == LLAMA_FTYPE_MOSTLY_Q8_0) { + throw std::runtime_error(format("this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1508)")); + } + } + + if (vocab_only) { + return; + } + + auto & ctx = model.ctx; + + size_t ctx_size; + size_t mmapped_size; + ml->calc_sizes(&ctx_size, &mmapped_size); + fprintf(stderr, "%s: ggml ctx size = %7.2f MB\n", __func__, ctx_size/1024.0/1024.0); + + // create the ggml context + { + model.buf.resize(ctx_size); + if (use_mlock) { + model.mlock_buf.init (model.buf.addr); + model.mlock_buf.grow_to(model.buf.size); + } + + struct ggml_init_params params = { + /*.mem_size =*/ model.buf.size, + /*.mem_buffer =*/ model.buf.addr, + /*.no_alloc =*/ ml->use_mmap, + }; + + model.ctx = ggml_init(params); + if (!model.ctx) { + throw std::runtime_error(format("ggml_init() failed")); + } + } + + (void) main_gpu; +#if defined(GGML_USE_CUBLAS) + fprintf(stderr, "%s: using CUDA for GPU acceleration\n", __func__); + ggml_cuda_set_main_device(main_gpu); +#define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU +#define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_GPU_SPLIT +#elif defined(GGML_USE_CLBLAST) + fprintf(stderr, "%s: using OpenCL for GPU acceleration\n", __func__); +#define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU +#define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_GPU +#else +#define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_CPU +#define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_CPU +#endif + + // prepare memory for the weights + size_t vram_weights = 0; + size_t vram_scratch = 0; + { + const uint32_t n_embd = hparams.n_embd; + const uint32_t n_embd_gqa = hparams.n_embd_gqa(); + const uint32_t n_layer = hparams.n_layer; + const uint32_t n_vocab = hparams.n_vocab; + + ml->ggml_ctx = ctx; + + model.tok_embeddings = ml->get_tensor("tok_embeddings.weight", {n_embd, n_vocab}, GGML_BACKEND_CPU); + + // "output" tensor + { + ggml_backend backend_norm; + ggml_backend backend_output; + if (n_gpu_layers > int(n_layer)) { // NOLINT + // norm is not performance relevant on its own but keeping it in VRAM reduces data copying + // on Windows however this is detrimental unless everything is on the GPU +#ifndef _WIN32 + backend_norm = low_vram ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD; +#else + backend_norm = low_vram || n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD; +#endif // _WIN32 + + backend_output = LLAMA_BACKEND_OFFLOAD_SPLIT; + } else { + backend_norm = GGML_BACKEND_CPU; + backend_output = GGML_BACKEND_CPU; + } + + model.norm = ml->get_tensor("norm.weight", {n_embd}, backend_norm); + model.output = ml->get_tensor("output.weight", {n_embd, n_vocab}, backend_output); + if (backend_norm == GGML_BACKEND_GPU) { + vram_weights += ggml_nbytes(model.norm); + } + if (backend_output == GGML_BACKEND_GPU_SPLIT) { + vram_weights += ggml_nbytes(model.output); + } + } + + const int i_gpu_start = n_layer - n_gpu_layers; + + model.layers.resize(n_layer); + for (uint32_t i = 0; i < n_layer; ++i) { + const ggml_backend backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD; // NOLINT + const ggml_backend backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD_SPLIT; // NOLINT + + auto & layer = model.layers[i]; + + std::string layers_i = "layers." + std::to_string(i); + + layer.attention_norm = ml->get_tensor(layers_i + ".attention_norm.weight", {n_embd}, backend); + + layer.wq = ml->get_tensor(layers_i + ".attention.wq.weight", {n_embd, n_embd}, backend_split); + layer.wk = ml->get_tensor(layers_i + ".attention.wk.weight", {n_embd, n_embd_gqa}, backend_split); + layer.wv = ml->get_tensor(layers_i + ".attention.wv.weight", {n_embd, n_embd_gqa}, backend_split); + layer.wo = ml->get_tensor(layers_i + ".attention.wo.weight", {n_embd, n_embd}, backend_split); + + layer.ffn_norm = ml->get_tensor(layers_i + ".ffn_norm.weight", {n_embd}, backend); + + layer.w1 = ml->get_tensor(layers_i + ".feed_forward.w1.weight", {n_embd, n_ff}, backend_split); + layer.w2 = ml->get_tensor(layers_i + ".feed_forward.w2.weight", { n_ff, n_embd}, backend_split); + layer.w3 = ml->get_tensor(layers_i + ".feed_forward.w3.weight", {n_embd, n_ff}, backend_split); + + if (backend == GGML_BACKEND_GPU) { + vram_weights += + ggml_nbytes(layer.attention_norm) + ggml_nbytes(layer.wq) + ggml_nbytes(layer.wk) + + ggml_nbytes(layer.wv) + ggml_nbytes(layer.wo) + ggml_nbytes(layer.ffn_norm) + + ggml_nbytes(layer.w1) + ggml_nbytes(layer.w2) + ggml_nbytes(layer.w3); + } + } + } + + ml->done_getting_tensors(); + + // print memory requirements + { + const size_t scale = memory_type == GGML_TYPE_F32 ? 2 : 1; + + // this is the total memory required to run the inference + const size_t mem_required = + ctx_size + + mmapped_size - vram_weights + // weights in VRAM not in memory + MEM_REQ_SCRATCH0(hparams.n_ctx).at(model.type) + + MEM_REQ_SCRATCH1().at(model.type) + + MEM_REQ_EVAL().at(model.type); + + // this is the memory required by one llama_state + const size_t mem_required_state = + scale*hparams.kv_size(); + + fprintf(stderr, "%s: mem required = %7.2f MB (+ %7.2f MB per state)\n", __func__, + mem_required / 1024.0 / 1024.0, mem_required_state / 1024.0 / 1024.0); + + (void) vram_scratch; + (void) n_batch; +#ifdef GGML_USE_CUBLAS + if (low_vram) { + fprintf(stderr, "%s: not allocating a VRAM scratch buffer due to low VRAM option\n", __func__); + ggml_cuda_set_scratch_size(0); // disable scratch + } else { + const size_t vram_scratch_base = VRAM_REQ_SCRATCH_BASE().at(model.type); + const size_t vram_scratch_per_context = VRAM_REQ_SCRATCH_PER_CONTEXT().at(model.type); + vram_scratch = n_batch * (vram_scratch_base + n_ctx * vram_scratch_per_context); + ggml_cuda_set_scratch_size(vram_scratch); + if (n_gpu_layers > 0) { + fprintf(stderr, "%s: allocating batch_size x (%zd kB + n_ctx x %zd B) = %zd MB VRAM for the scratch buffer\n", + __func__, vram_scratch_base / kB, vram_scratch_per_context, + (vram_scratch + MB - 1) / MB); // round up + } + } +#endif // GGML_USE_CUBLAS + +#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) + const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer)); + + fprintf(stderr, "%s: offloading %d repeating layers to GPU\n", __func__, n_gpu); + if (n_gpu_layers > (int) hparams.n_layer) { + fprintf(stderr, "%s: offloading non-repeating layers to GPU\n", __func__); + } + size_t vram_kv_cache = 0; + +#ifdef GGML_USE_CUBLAS + const int max_backend_supported_layers = hparams.n_layer + 3; + const int max_offloadable_layers = low_vram ? hparams.n_layer + 1 : hparams.n_layer + 3; + if (n_gpu_layers > (int) hparams.n_layer + 1) { + if (low_vram) { + fprintf(stderr, "%s: cannot offload v cache to GPU due to low VRAM option\n", __func__); + } else { + fprintf(stderr, "%s: offloading v cache to GPU\n", __func__); + vram_kv_cache += hparams.kv_size() / 2; + } + } + if (n_gpu_layers > (int) hparams.n_layer + 2) { + if (low_vram) { + fprintf(stderr, "%s: cannot offload k cache to GPU due to low VRAM option\n", __func__); + } else { + fprintf(stderr, "%s: offloading k cache to GPU\n", __func__); + vram_kv_cache += hparams.kv_size() / 2; + } + } +#elif defined(GGML_USE_CLBLAST) + const int max_backend_supported_layers = hparams.n_layer + 1; + const int max_offloadable_layers = hparams.n_layer + 1; +#endif // GGML_USE_CUBLAS + + fprintf(stderr, "%s: offloaded %d/%d layers to GPU\n", + __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers); + fprintf(stderr, "%s: total VRAM used: %zu MB\n", + __func__, (vram_weights + vram_scratch + vram_kv_cache + MB - 1) / MB); // round up +#else + (void) n_gpu_layers; +#endif // defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) + } + + // populate `tensors_by_name` + for (llama_load_tensor & lt : ml->tensors_map.tensors) { + model.tensors_by_name.emplace_back(lt.name, lt.ggml_tensor); + } + + (void) tensor_split; +#if defined(GGML_USE_CUBLAS) + { + ggml_cuda_set_tensor_split(tensor_split); + } +#endif + + ml->load_all_data(progress_callback, progress_callback_user_data, use_mlock ? &model.mlock_mmap : NULL); + + if (progress_callback) { + progress_callback(1.0f, progress_callback_user_data); + } + + model.mapping = std::move(ml->mapping); + + // loading time will be recalculate after the first eval, so + // we take page faults deferred by mmap() into consideration + model.t_load_us = ggml_time_us() - model.t_start_us; +} + +static bool llama_model_load( + const std::string & fname, + llama_model & model, + llama_vocab & vocab, + int n_ctx, + int n_batch, + int n_gqa, + float rms_norm_eps, + int n_gpu_layers, + int main_gpu, + const float * tensor_split, + float rope_freq_base, + float rope_freq_scale, + bool low_vram, + ggml_type memory_type, + bool use_mmap, + bool use_mlock, + bool vocab_only, + llama_progress_callback progress_callback, + void *progress_callback_user_data) { + try { + llama_model_load_internal(fname, model, vocab, n_ctx, n_batch, n_gqa, rms_norm_eps, n_gpu_layers, main_gpu, tensor_split, rope_freq_base, rope_freq_scale, low_vram, memory_type, + use_mmap, use_mlock, vocab_only, progress_callback, progress_callback_user_data); + return true; + } catch (const std::exception & err) { + fprintf(stderr, "error loading model: %s\n", err.what()); + return false; + } +} + +// evaluate the transformer +// +// - lctx: llama context +// - tokens: new batch of tokens to process +// - embd embeddings input +// - n_tokens number of tokens +// - n_past: the context size so far +// - n_threads: number of threads to use +// +static bool llama_eval_internal( + llama_context & lctx, + const llama_token * tokens, + const float * embd, + int n_tokens, + int n_past, + int n_threads, + const char * cgraph_fname) { + + GGML_ASSERT((!tokens && embd) || (tokens && !embd)); + +#ifdef GGML_USE_MPI + ggml_mpi_eval_init(lctx.ctx_mpi, &n_tokens, &n_past, &n_threads); +#endif + + const int64_t t_start_us = ggml_time_us(); + + const int N = n_tokens; + + const auto & model = lctx.model; + const auto & hparams = model.hparams; + + const auto & kv_self = lctx.kv_self; + + GGML_ASSERT(!!kv_self.ctx); + + const int64_t n_embd = hparams.n_embd; + const int64_t n_layer = hparams.n_layer; + const int64_t n_ctx = hparams.n_ctx; + const int64_t n_head = hparams.n_head; + const int64_t n_head_kv = hparams.n_head_kv; + const int64_t n_embd_head = hparams.n_embd_head(); + const int64_t n_vocab = hparams.n_vocab; + const int64_t n_embd_gqa = hparams.n_embd_gqa(); + + + GGML_ASSERT(n_embd_head == hparams.n_rot); + + const float freq_base = hparams.rope_freq_base; + const float freq_scale = hparams.rope_freq_scale; + const float rms_norm_eps = hparams.f_rms_norm_eps; + + const int n_gpu_layers = model.n_gpu_layers; + + auto & mem_per_token = lctx.mem_per_token; + auto & buf_compute = lctx.buf_compute; + + struct ggml_init_params params = { + /*.mem_size =*/ buf_compute.size, + /*.mem_buffer =*/ buf_compute.addr, + /*.no_alloc =*/ false, + }; + + struct ggml_context * ctx0 = ggml_init(params); + + ggml_cgraph * gf = ggml_new_graph(ctx0); + + // for big prompts, if BLAS is enabled, it is better to use only one thread + // otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance + n_threads = N >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas() ? 1 : n_threads; + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + if (tokens) { + struct ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); + memcpy(inp_tokens->data, tokens, N*ggml_element_size(inp_tokens)); + ggml_set_name(inp_tokens, "inp_tokens"); + + inpL = ggml_get_rows(ctx0, model.tok_embeddings, inp_tokens); + } else { +#ifdef GGML_USE_MPI + GGML_ASSERT(false && "not implemented"); +#endif + + inpL = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N); + memcpy(inpL->data, embd, N * n_embd * ggml_element_size(inpL)); + } + + const int i_gpu_start = n_layer - n_gpu_layers; + (void) i_gpu_start; + + // offload functions set the tensor output backend to GPU + // tensors are GPU-accelerated if any input or the output has been offloaded + // + // with the low VRAM option VRAM scratch is disabled in llama_load_model_internal + // in that case ggml_cuda_assign_buffers has no effect + offload_func_t offload_func_nr = llama_nop; // nr = non-repeating + offload_func_t offload_func_kq = llama_nop; + offload_func_t offload_func_v = llama_nop; + +#ifdef GGML_USE_CUBLAS + if (n_gpu_layers > n_layer) { + offload_func_nr = ggml_cuda_assign_buffers; + } + if (n_gpu_layers > n_layer + 1) { + offload_func_v = ggml_cuda_assign_buffers; + } + if (n_gpu_layers > n_layer + 2) { + offload_func_kq = ggml_cuda_assign_buffers; + } +#endif // GGML_USE_CUBLAS + + for (int il = 0; il < n_layer; ++il) { + ggml_format_name(inpL, "layer_inp_%d", il); + + offload_func_t offload_func = llama_nop; + +#ifdef GGML_USE_CUBLAS + if (il >= i_gpu_start) { + offload_func = ggml_cuda_assign_buffers; + } +#endif // GGML_USE_CUBLAS + + struct ggml_tensor * inpSA = inpL; + + lctx.use_buf(ctx0, 0); + + // norm + { + cur = ggml_rms_norm(ctx0, inpL, rms_norm_eps); + offload_func(cur); + ggml_set_name(cur, "rms_norm_0"); + + // cur = cur*attention_norm(broadcasted) + cur = ggml_mul(ctx0, cur, model.layers[il].attention_norm); + offload_func(cur); + ggml_set_name(cur, "attention_norm_0"); + } + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * tmpk = ggml_mul_mat(ctx0, model.layers[il].wk, cur); + offload_func_kq(tmpk); + ggml_set_name(tmpk, "tmpk"); + + struct ggml_tensor * tmpq = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + offload_func_kq(tmpq); + ggml_set_name(tmpq, "tmpq"); + + struct ggml_tensor * Kcur = ggml_rope_custom_inplace(ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, N), n_past, n_embd_head, 0, 0, freq_base, freq_scale); + offload_func_kq(Kcur); + ggml_set_name(Kcur, "Kcur"); + + struct ggml_tensor * Qcur = ggml_rope_custom_inplace(ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, N), n_past, n_embd_head, 0, 0, freq_base, freq_scale); + offload_func_kq(Qcur); + ggml_set_name(Qcur, "Qcur"); + + // store key and value to memory + { + // compute the transposed [N, n_embd] V matrix + + struct ggml_tensor * tmpv = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + offload_func_v(tmpv); + ggml_set_name(tmpv, "tmpv"); + + struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_reshape_2d(ctx0, tmpv, n_embd_gqa, N)); + offload_func_v(Vcur); + ggml_set_name(Vcur, "Vcur"); + + struct ggml_tensor * k = ggml_view_1d(ctx0, kv_self.k, N*n_embd_gqa, (ggml_element_size(kv_self.k)*n_embd_gqa)*(il*n_ctx + n_past)); + offload_func_kq(k); + ggml_set_name(k, "k"); + + struct ggml_tensor * v = ggml_view_2d(ctx0, kv_self.v, N, n_embd_gqa, + ( n_ctx)*ggml_element_size(kv_self.v), + (il*n_ctx)*ggml_element_size(kv_self.v)*n_embd_gqa + n_past*ggml_element_size(kv_self.v)); + offload_func_v(v); + ggml_set_name(v, "v"); + + // important: storing RoPE-ed version of K in the KV cache! + ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k)); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v)); + } + + struct ggml_tensor * Q = + ggml_permute(ctx0, + Qcur, + 0, 2, 1, 3); + offload_func_kq(Q); + ggml_set_name(Q, "Q"); + + struct ggml_tensor * K = + ggml_permute(ctx0, + ggml_reshape_3d(ctx0, + ggml_view_1d(ctx0, kv_self.k, (n_past + N)*n_embd_gqa, il*n_ctx*ggml_element_size(kv_self.k)*n_embd_gqa), + n_embd_head, n_head_kv, n_past + N), + 0, 2, 1, 3); + offload_func_kq(K); + ggml_set_name(K, "K"); + + // K * Q + struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); + offload_func_kq(KQ); + ggml_set_name(KQ, "KQ"); + + // KQ_scaled = KQ / sqrt(n_embd_head) + struct ggml_tensor * KQ_scale = ggml_new_f32(ctx0, 1.0f/sqrtf(float(n_embd)/n_head)); + ggml_set_name(KQ_scale, "1/sqrt(n_embd_head)"); + + // KQ_scaled shape [n_past + N, N, n_head, 1] + struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, KQ_scale); + offload_func_kq(KQ_scaled); + ggml_set_name(KQ_scaled, "KQ_scaled"); + + // KQ_masked = mask_past(KQ_scaled) + struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past); + offload_func_kq(KQ_masked); + ggml_set_name(KQ_masked, "KQ_masked"); + + // KQ = soft_max(KQ_masked) + struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked); + offload_func_v(KQ_soft_max); + ggml_set_name(KQ_soft_max, "KQ_soft_max"); + + // split cached V into n_head heads + struct ggml_tensor * V = + ggml_view_3d(ctx0, kv_self.v, + n_past + N, n_embd_head, n_head_kv, + n_ctx*ggml_element_size(kv_self.v), + n_ctx*ggml_element_size(kv_self.v)*n_embd_head, + n_ctx*ggml_element_size(kv_self.v)*n_embd_gqa*il); + offload_func_v(V); + ggml_set_name(V, "V"); + +#if 1 + struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max); + offload_func_v(KQV); + ggml_set_name(KQV, "KQV"); +#else + // make V contiguous in memory to speed up the matmul, however we waste time on the copy + // on M1 this is faster for the perplexity computation, but ~5% slower for the single-token generation + // is there a better way? + struct ggml_tensor * V_cont = ggml_cpy(ctx0, V, ggml_new_tensor_3d(ctx0, kv_self.v->type, n_past + N, n_embd_head, n_head)); + struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_cont, KQ_soft_max); +#endif + + // KQV_merged = KQV.permute(0, 2, 1, 3) + struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3); + offload_func_v(KQV_merged); + ggml_set_name(KQV_merged, "KQV_merged"); + + // cur = KQV_merged.contiguous().view(n_embd, N) + cur = ggml_cpy(ctx0, + KQV_merged, + ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N)); + offload_func_v(cur); + ggml_set_name(cur, "KQV_merged_contiguous"); + + // projection (no bias) + cur = ggml_mul_mat(ctx0, + model.layers[il].wo, + cur); + offload_func(cur); + ggml_set_name(cur, "result_wo"); + } + + lctx.use_buf(ctx0, 1); + + struct ggml_tensor * inpFF = ggml_add(ctx0, cur, inpSA); + offload_func(inpFF); + ggml_set_name(inpFF, "inpFF"); + + // feed-forward network + { + // norm + { + cur = ggml_rms_norm(ctx0, inpFF, rms_norm_eps); + offload_func(cur); + ggml_set_name(cur, "rms_norm_1"); + + // cur = cur*ffn_norm(broadcasted) + cur = ggml_mul(ctx0, cur, model.layers[il].ffn_norm); + offload_func(cur); + ggml_set_name(cur, "ffn_norm"); + } + + struct ggml_tensor * tmp = ggml_mul_mat(ctx0, + model.layers[il].w3, + cur); + offload_func(tmp); + ggml_set_name(tmp, "result_w3"); + + cur = ggml_mul_mat(ctx0, + model.layers[il].w1, + cur); + offload_func(cur); + ggml_set_name(cur, "result_w1"); + + // SILU activation + cur = ggml_silu(ctx0, cur); + offload_func(cur); + ggml_set_name(cur, "silu"); + + cur = ggml_mul(ctx0, cur, tmp); + offload_func(cur); + ggml_set_name(cur, "silu_x_result_w3"); + + cur = ggml_mul_mat(ctx0, + model.layers[il].w2, + cur); + offload_func(cur); + ggml_set_name(cur, "result_w2"); + } + + cur = ggml_add(ctx0, cur, inpFF); + offload_func(cur); + ggml_set_name(cur, "inpFF_+_result_w2"); + + // input for next layer + inpL = cur; + } + + lctx.use_buf(ctx0, 0); + + // used at the end to optionally extract the embeddings + struct ggml_tensor * embeddings = NULL; + + // norm + { + cur = ggml_rms_norm(ctx0, inpL, rms_norm_eps); + offload_func_nr(cur); + ggml_set_name(cur, "rms_norm_2"); + + // cur = cur*norm(broadcasted) + cur = ggml_mul(ctx0, cur, model.norm); + // offload_func_nr(cur); // TODO CPU + GPU mirrored backend + ggml_set_name(cur, "result_norm"); + + embeddings = cur; + } + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + ggml_set_name(cur, "result_output"); + + lctx.use_buf(ctx0, -1); + + // logits -> probs + //cur = ggml_soft_max_inplace(ctx0, cur); + + // run the computation + ggml_build_forward_expand(gf, cur); + + // fprintf(stderr, "graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf.n_nodes, gf.n_leafs); + +#if GGML_USE_MPI + ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer); +#endif + +#ifdef GGML_USE_METAL + if (lctx.ctx_metal && N == 1) { + if (!ggml_metal_if_optimized(lctx.ctx_metal)) { + ggml_metal_graph_find_concurrency(lctx.ctx_metal, gf); + } + ggml_metal_set_n_cb (lctx.ctx_metal, n_threads); + ggml_metal_graph_compute(lctx.ctx_metal, gf); + ggml_metal_get_tensor (lctx.ctx_metal, cur); + } else { + // IMPORTANT: + // Since we don't have efficient Matrix x Matrix Metal multiplication yet, we fallback to vanilla + // ggml_graph_compute(). It uses Apple's Accelerate CBLAS API which takes advantage of the ANE or the AMX + // coprocessor. + // + // When we implement Matrix x Matrix Metal multiplication, we can avoid this branch. + // But for now, we have focused only on Matrix x Vector Metal multiplication. + // + // TODO: avoid these syncs via shared memory (ref #1696) + // + if (lctx.ctx_metal) { + // We need to sync the GPU KV cache with the CPU KV cache + ggml_metal_get_tensor(lctx.ctx_metal, kv_self.k); + ggml_metal_get_tensor(lctx.ctx_metal, kv_self.v); + } + + ggml_graph_compute_helper(lctx.work_buffer, gf, n_threads); + } +#else + ggml_graph_compute_helper(lctx.work_buffer, gf, n_threads); +#endif + +#if GGML_USE_MPI + ggml_mpi_graph_compute_post(lctx.ctx_mpi, gf, n_layer); +#endif + + // update kv token count + lctx.kv_self.n = n_past + N; + + struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; + + if (cgraph_fname) { + ggml_graph_export(gf, cgraph_fname); + } + +#ifdef GGML_PERF + // print timing information per ggml operation (for debugging purposes) + // requires GGML_PERF to be defined + ggml_graph_print(gf); +#endif + + // plot the computation graph in dot format (for debugging purposes) + //if (n_past%100 == 0) { + // ggml_graph_dump_dot(gf, NULL, "llama.dot"); + //} + + // extract logits + { + auto & logits_out = lctx.logits; + + if (lctx.logits_all) { + logits_out.resize(n_vocab * N); + memcpy(logits_out.data(), (float *) ggml_get_data(res), sizeof(float)*n_vocab*N); + } else { + // return result for just the last token + logits_out.resize(n_vocab); + memcpy(logits_out.data(), (float *) ggml_get_data(res) + (n_vocab*(N-1)), sizeof(float)*n_vocab); + } + } + + // extract embeddings + if (!lctx.embedding.empty()) { + auto & embedding_out = lctx.embedding; + + embedding_out.resize(n_embd); + memcpy(embedding_out.data(), (float *) ggml_get_data(embeddings) + (n_embd*(N - 1)), sizeof(float)*n_embd); + } + + if (mem_per_token == 0) { + mem_per_token = ggml_used_mem(ctx0)/N; + } + +#if 0 + printf("\n%s: used_mem: eval ctx %.3f MB, scratch %.3f MB %.3f MB, work buf %.3f MB, n_past = %d, N = %d\n", __func__, + ggml_used_mem(ctx0)/1024.0/1024.0, + lctx.get_buf_max_mem(0)/1024.0/1024.0, + lctx.get_buf_max_mem(1)/1024.0/1024.0, + lctx.work_buffer.size()/1024.0/1024.0, + n_past, N); +#endif + + ggml_free(ctx0); + + // measure the performance only for the single-token evals + if (N == 1) { + lctx.t_eval_us += ggml_time_us() - t_start_us; + lctx.n_eval++; + } + else if (N > 1) { + lctx.t_p_eval_us += ggml_time_us() - t_start_us; + lctx.n_p_eval += N; + } + + return true; +} + +// +// tokenizer +// + +static size_t utf8_len(char src) { + const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; + uint8_t highbits = static_cast(src) >> 4; + return lookup[highbits]; +} + +struct llama_sp_symbol { + using index = int; + index prev; + index next; + const char * text; + size_t n; +}; + +static_assert(std::is_trivially_copyable::value, "llama_sp_symbol is not trivially copyable"); + +struct llama_sp_bigram { + struct comparator { + bool operator()(llama_sp_bigram & l, llama_sp_bigram & r) { + return (l.score < r.score) || (l.score == r.score && l.left > r.left); + } + }; + using queue_storage = std::vector; + using queue = std::priority_queue; + llama_sp_symbol::index left; + llama_sp_symbol::index right; + float score; + size_t size; +}; + +// original implementation: +// https://github.com/ggerganov/llama.cpp/commit/074bea2eb1f1349a0118239c4152914aecaa1be4 +struct llama_tokenizer { + llama_tokenizer(const llama_vocab & vocab): vocab_(vocab) {} + + void tokenize(const std::string & text, std::vector & output) { + // split string into utf8 chars + int index = 0; + size_t offs = 0; + while (offs < text.size()) { + llama_sp_symbol sym; + size_t char_len = std::min(text.size() - offs, utf8_len(text[offs])); + sym.text = text.c_str() + offs; + sym.n = char_len; + offs += char_len; + sym.prev = index - 1; + sym.next = offs == text.size() ? -1 : index + 1; + index++; + symbols_.emplace_back(sym); + } + + // seed the work queue with all possible 2-character tokens. + for (size_t i = 1; i < symbols_.size(); ++i) { + try_add_bigram(i - 1, i); + } + + // keep substituting the highest frequency pairs for as long as we can. + while (!work_queue_.empty()) { + auto bigram = work_queue_.top(); + work_queue_.pop(); + + auto & left_sym = symbols_[bigram.left]; + auto & right_sym = symbols_[bigram.right]; + + // if one of the symbols already got merged, skip it. + if (left_sym.n == 0 || right_sym.n == 0 || + left_sym.n + right_sym.n != bigram.size) { + continue; + } + + // merge the right sym into the left one + left_sym.n += right_sym.n; + right_sym.n = 0; + + //printf("left = '%*s' size = %zu\n", (int) left_sym.n, left_sym.text, bigram.size); + + // remove the right sym from the chain + left_sym.next = right_sym.next; + if (right_sym.next >= 0) { + symbols_[right_sym.next].prev = bigram.left; + } + + // find more substitutions + try_add_bigram(left_sym.prev, bigram.left); + try_add_bigram(bigram.left, left_sym.next); + } + + for (int i = 0; i != -1; i = symbols_[i].next) { + auto & symbol = symbols_[i]; + auto token = vocab_.token_to_id.find(std::string(symbol.text, symbol.n)); + + if (token == vocab_.token_to_id.end()) { + // output any symbols that did not form tokens as bytes. + for (int j = 0; j < (int) symbol.n; ++j) { + llama_vocab::id token_id = static_cast(symbol.text[j]) + 3; + output.push_back(token_id); + } + } else { + output.push_back((*token).second); + } + } + } + +private: + void try_add_bigram(int left, int right) { + if (left == -1 || right == -1) { + return; + } + + const std::string text = std::string(symbols_[left].text, symbols_[left].n + symbols_[right].n); + auto token = vocab_.token_to_id.find(text); + + if (token == vocab_.token_to_id.end()) { + return; + } + + if (static_cast((*token).second) >= vocab_.id_to_token.size()) { + return; + } + + const auto &tok_score = vocab_.id_to_token[(*token).second]; + + llama_sp_bigram bigram; + bigram.left = left; + bigram.right = right; + bigram.score = tok_score.score; + bigram.size = text.size(); + work_queue_.push(bigram); + } + + const llama_vocab & vocab_; + std::vector symbols_; + llama_sp_bigram::queue work_queue_; +}; + +static std::vector llama_tokenize(const llama_vocab & vocab, const std::string & text, bool bos) { + llama_tokenizer tokenizer(vocab); + std::vector output; + + if (text.empty()) { + return output; + } + + if (bos) { + output.push_back(llama_token_bos()); + } + + tokenizer.tokenize(text, output); + return output; +} + +// +// grammar - internal +// + +struct llama_grammar { + const std::vector> rules; + std::vector> stacks; +}; + +struct llama_grammar_candidate { + size_t index; + const uint32_t * code_points; +}; + +// NOTE: assumes valid utf8 (but checks for overrun) +// adds a terminating 0 for use as pointer +std::vector decode_utf8(const char * src) { + static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; + const char * pos = src; + std::vector code_points; + while (*pos != 0) { + uint8_t first_byte = static_cast(*pos); + uint8_t highbits = first_byte >> 4; + int len = lookup[highbits]; + uint8_t mask = (1 << (8 - len)) - 1; + uint32_t value = first_byte & mask; + const char * end = pos + len; // may overrun! + ++pos; + for ( ; pos < end && *pos != 0; ++pos) { + value = (value << 6) + (static_cast(*pos) & 0x3F); + } + code_points.push_back(value); + } + code_points.push_back(0); + return code_points; +} + +// returns true iff pos points to the end of one of the definitions of a rule +static bool llama_grammar_is_end_of_sequence(const llama_grammar_element * pos) { + switch (pos->type) { + case LLAMA_GRETYPE_END: return true; + case LLAMA_GRETYPE_ALT: return true; + default: return false; + } +} + +// returns true iff chr satisfies the char range at pos (regular or inverse range) +// asserts that pos is pointing to a char range element +static std::pair llama_grammar_match_char( + const llama_grammar_element * pos, + const uint32_t chr) { + + bool found = false; + bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR; + GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT); + + do { + if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) { + // inclusive range, e.g. [a-z] + found = found || (pos->value <= chr && chr <= pos[1].value); + pos += 2; + } else { + // exact char match, e.g. [a] or "a" + found = found || pos->value == chr; + pos += 1; + } + } while (pos->type == LLAMA_GRETYPE_CHAR_ALT); + + return std::make_pair(found == is_positive_char, pos); +} + +// transforms a grammar pushdown stack into N possible stacks, all ending +// at a character range (terminal element) +static void llama_grammar_advance_stack( + const std::vector> & rules, + const std::vector & stack, + std::vector> & new_stacks) { + + if (stack.empty()) { + new_stacks.push_back(stack); + return; + } + + const llama_grammar_element * pos = stack.back(); + + switch (pos->type) { + case LLAMA_GRETYPE_RULE_REF: { + const size_t rule_id = static_cast(pos->value); + const llama_grammar_element * subpos = rules[rule_id].data(); + do { + // init new stack without the top (pos) + std::vector new_stack(stack.begin(), stack.end() - 1); + if (!llama_grammar_is_end_of_sequence(pos + 1)) { + // if this rule ref is followed by another element, add that to stack + new_stack.push_back(pos + 1); + } + if (!llama_grammar_is_end_of_sequence(subpos)) { + // if alternate is nonempty, add to stack + new_stack.push_back(subpos); + } + llama_grammar_advance_stack(rules, new_stack, new_stacks); + while (!llama_grammar_is_end_of_sequence(subpos)) { + // scan to end of alternate def + subpos++; + } + if (subpos->type == LLAMA_GRETYPE_ALT) { + // there's another alternate def of this rule to process + subpos++; + } else { + break; + } + } while (true); + break; + } + case LLAMA_GRETYPE_CHAR: + case LLAMA_GRETYPE_CHAR_NOT: + new_stacks.push_back(stack); + break; + default: + // end of alternate (LLAMA_GRETYPE_END, LLAMA_GRETYPE_ALT) or middle of char range + // (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER); stack should never be left on + // those + GGML_ASSERT(false); + } +} + +// takes a set of possible pushdown stacks on a grammar, which are required to +// be positioned at a character range (see `llama_grammar_advance_stack`), and +// produces the N possible stacks if the given char is accepted at those +// positions +static std::vector> llama_grammar_accept( + const std::vector> & rules, + const std::vector> & stacks, + const uint32_t chr) { + + std::vector> new_stacks; + + for (const auto & stack : stacks) { + if (stack.empty()) { + continue; + } + + auto match = llama_grammar_match_char(stack.back(), chr); + if (match.first) { + const llama_grammar_element * pos = match.second; + + // update top of stack to next element, if any + std::vector new_stack(stack.begin(), stack.end() - 1); + if (!llama_grammar_is_end_of_sequence(pos)) { + new_stack.push_back(pos); + } + llama_grammar_advance_stack(rules, new_stack, new_stacks); + } + } + + return new_stacks; +} + +static std::vector llama_grammar_reject_candidates( + const std::vector> & rules, + const std::vector> & stacks, + const std::vector & candidates); + +static std::vector llama_grammar_reject_candidates_for_stack( + const std::vector> & rules, + const std::vector & stack, + const std::vector & candidates) { + + std::vector rejects; + + if (stack.empty()) { + // accept nothing; EOS is handled elsewhere + rejects.insert(rejects.end(), candidates.begin(), candidates.end()); + return rejects; + } + + const llama_grammar_element * stack_pos = stack.back(); + + std::vector next_candidates; + for (auto tok : candidates) { + if (llama_grammar_match_char(stack_pos, tok.code_points[0]).first) { + if (tok.code_points[1] != 0) { + next_candidates.push_back({ tok.index, tok.code_points + 1 }); + } + } else { + rejects.push_back(tok); + } + } + + auto stack_pos_after = llama_grammar_match_char(stack_pos, 0).second; + + // update top of stack to next element, if any + std::vector stack_after(stack.begin(), stack.end() - 1); + if (!llama_grammar_is_end_of_sequence(stack_pos_after)) { + stack_after.push_back(stack_pos_after); + } + std::vector> next_stacks; + llama_grammar_advance_stack(rules, stack_after, next_stacks); + + auto next_rejects = llama_grammar_reject_candidates(rules, next_stacks, next_candidates); + for (auto tok : next_rejects) { + rejects.push_back({ tok.index, tok.code_points - 1 }); + } + + return rejects; +} + +static std::vector llama_grammar_reject_candidates( + const std::vector> & rules, + const std::vector> & stacks, + const std::vector & candidates) { + GGML_ASSERT(!stacks.empty()); // REVIEW + + if (candidates.empty()) { + return std::vector(); + } + + auto rejects = llama_grammar_reject_candidates_for_stack(rules, stacks.front(), candidates); + + for (size_t i = 1, size = stacks.size(); i < size; ++i) { + rejects = llama_grammar_reject_candidates_for_stack(rules, stacks[i], rejects); + } + return rejects; +} + +// +// grammar - external +// + +struct llama_grammar * llama_grammar_init( + const llama_grammar_element ** rules, + size_t n_rules, + size_t start_rule_index) { + const llama_grammar_element * pos; + + // copy rule definitions into vectors + std::vector> vec_rules(n_rules); + for (size_t i = 0; i < n_rules; i++) { + for (pos = rules[i]; pos->type != LLAMA_GRETYPE_END; pos++) { + vec_rules[i].push_back(*pos); + } + vec_rules[i].push_back({LLAMA_GRETYPE_END, 0}); + } + + // loop over alternates of start rule to build initial stacks + std::vector> stacks; + pos = rules[start_rule_index]; + do { + std::vector stack; + if (!llama_grammar_is_end_of_sequence(pos)) { + // if alternate is nonempty, add to stack + stack.push_back(pos); + } + llama_grammar_advance_stack(vec_rules, stack, stacks); + while (!llama_grammar_is_end_of_sequence(pos)) { + // scan to end of alternate def + pos++; + } + if (pos->type == LLAMA_GRETYPE_ALT) { + // there's another alternate def of this rule to process + pos++; + } else { + break; + } + } while (true); + + return new llama_grammar{ std::move(vec_rules), std::move(stacks) }; +} + +void llama_grammar_free(struct llama_grammar * grammar) { + delete grammar; +} + +// +// sampling +// + +void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates) { + assert(candidates->size > 0); + + const int64_t t_start_sample_us = ggml_time_us(); + + // Sort the logits in descending order + if (!candidates->sorted) { + std::sort(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) { + return a.logit > b.logit; + }); + candidates->sorted = true; + } + + float max_l = candidates->data[0].logit; + float cum_sum = 0.0f; + for (size_t i = 0; i < candidates->size; ++i) { + float p = expf(candidates->data[i].logit - max_l); + candidates->data[i].p = p; + cum_sum += p; + } + for (size_t i = 0; i < candidates->size; ++i) { + candidates->data[i].p /= cum_sum; + } + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep) { + const int64_t t_start_sample_us = ggml_time_us(); + + k = std::max(k, (int) min_keep); + k = std::min(k, (int) candidates->size); + + // Sort scores in descending order + if (!candidates->sorted) { + auto comp = [](const llama_token_data & a, const llama_token_data & b) { + return a.logit > b.logit; + }; + if (k == (int) candidates->size) { + std::sort(candidates->data, candidates->data + candidates->size, comp); + } else { + std::partial_sort(candidates->data, candidates->data + k, candidates->data + candidates->size, comp); + } + candidates->sorted = true; + } + candidates->size = k; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) { + if (p >= 1.0f) { + return; + } + + llama_sample_softmax(ctx, candidates); + + const int64_t t_start_sample_us = ggml_time_us(); + + // Compute the cumulative probabilities + float cum_sum = 0.0f; + size_t last_idx = candidates->size; + + for (size_t i = 0; i < candidates->size; ++i) { + cum_sum += candidates->data[i].p; + + // Check if the running sum is at least p or if we have kept at least min_keep tokens + // we set the last index to i+1 to indicate that the current iterate should be included in the set + if (cum_sum >= p && i + 1 >= min_keep) { + last_idx = i + 1; + break; + } + } + + // Resize the output vector to keep only the top-p tokens + candidates->size = last_idx; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep) { + if (z >= 1.0f || candidates->size <= 2) { + return; + } + + llama_sample_softmax(nullptr, candidates); + const int64_t t_start_sample_us = ggml_time_us(); + + // Compute the first and second derivatives + std::vector first_derivatives(candidates->size - 1); + std::vector second_derivatives(candidates->size - 2); + + for (size_t i = 0; i < first_derivatives.size(); ++i) { + first_derivatives[i] = candidates->data[i].p - candidates->data[i + 1].p; + } + for (size_t i = 0; i < second_derivatives.size(); ++i) { + second_derivatives[i] = first_derivatives[i] - first_derivatives[i + 1]; + } + + // Calculate absolute value of second derivatives + for (size_t i = 0; i < second_derivatives.size(); ++i) { + second_derivatives[i] = abs(second_derivatives[i]); + } + + // Normalize the second derivatives + { + const float second_derivatives_sum = std::accumulate(second_derivatives.begin(), second_derivatives.end(), 0.0f); + + if (second_derivatives_sum > 1e-6f) { + for (float & value : second_derivatives) { + value /= second_derivatives_sum; + } + } else { + for (float & value : second_derivatives) { + value = 1.0f / second_derivatives.size(); + } + } + } + + float cum_sum = 0.0f; + size_t last_idx = candidates->size; + for (size_t i = 0; i < second_derivatives.size(); ++i) { + cum_sum += second_derivatives[i]; + + // Check if the running sum is greater than z or if we have kept at least min_keep tokens + if (cum_sum > z && i >= min_keep) { + last_idx = i; + break; + } + } + + // Resize the output vector to keep only the tokens above the tail location + candidates->size = last_idx; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + + +void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) { + // Reference implementation: + // https://github.com/huggingface/transformers/compare/main...cimeister:typical-sampling:typical-pr + if (p >= 1.0f) { + return; + } + + // Compute the softmax of logits and calculate entropy + llama_sample_softmax(nullptr, candidates); + + const int64_t t_start_sample_us = ggml_time_us(); + + float entropy = 0.0f; + for (size_t i = 0; i < candidates->size; ++i) { + entropy += -candidates->data[i].p * logf(candidates->data[i].p); + } + + // Compute the absolute difference between negative log probability and entropy for each candidate + std::vector shifted_scores; + for (size_t i = 0; i < candidates->size; ++i) { + float shifted_score = fabsf(-logf(candidates->data[i].p) - entropy); + shifted_scores.push_back(shifted_score); + } + + // Sort tokens based on the shifted_scores and their corresponding indices + std::vector indices(candidates->size); + std::iota(indices.begin(), indices.end(), 0); + + std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) { + return shifted_scores[a] < shifted_scores[b]; + }); + + // Compute the cumulative probabilities + float cum_sum = 0.0f; + size_t last_idx = indices.size(); + + for (size_t i = 0; i < indices.size(); ++i) { + size_t idx = indices[i]; + cum_sum += candidates->data[idx].p; + + // Check if the running sum is greater than typical or if we have kept at least min_keep tokens + if (cum_sum > p && i >= min_keep - 1) { + last_idx = i + 1; + break; + } + } + + // Resize the output vector to keep only the locally typical tokens + std::vector new_candidates; + for (size_t i = 0; i < last_idx; ++i) { + size_t idx = indices[i]; + new_candidates.push_back(candidates->data[idx]); + } + + // Replace the data in candidates with the new_candidates data + std::copy(new_candidates.begin(), new_candidates.end(), candidates->data); + candidates->size = new_candidates.size(); + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates_p, float temp) { + const int64_t t_start_sample_us = ggml_time_us(); + + for (size_t i = 0; i < candidates_p->size; ++i) { + candidates_p->data[i].logit /= temp; + } + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_repetition_penalty(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float penalty) { + if (last_tokens_size == 0 || penalty == 1.0f) { + return; + } + + const int64_t t_start_sample_us = ggml_time_us(); + + for (size_t i = 0; i < candidates->size; ++i) { + const auto * token_iter = std::find(last_tokens, last_tokens + last_tokens_size, candidates->data[i].id); + if (token_iter == last_tokens + last_tokens_size) { + continue; + } + + // The academic publication that described this technique actually just only divided, but that would cause tokens with negative logits to become more likely, which is obviously wrong. + // This is common fix for this problem, which is to multiply by the penalty instead of dividing. + if (candidates->data[i].logit <= 0) { + candidates->data[i].logit *= penalty; + } else { + candidates->data[i].logit /= penalty; + } + } + + candidates->sorted = false; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_frequency_and_presence_penalties(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens_p, size_t last_tokens_size, float alpha_frequency, float alpha_presence) { + if (last_tokens_size == 0 || (alpha_frequency == 0.0f && alpha_presence == 0.0f)) { + return; + } + + const int64_t t_start_sample_us = ggml_time_us(); + + // Create a frequency map to count occurrences of each token in last_tokens + std::unordered_map token_count; + for (size_t i = 0; i < last_tokens_size; ++i) { + token_count[last_tokens_p[i]]++; + } + + // Apply frequency and presence penalties to the candidates + for (size_t i = 0; i < candidates->size; ++i) { + auto token_iter = token_count.find(candidates->data[i].id); + if (token_iter == token_count.end()) { + continue; + } + + int count = token_iter->second; + candidates->data[i].logit -= float(count) * alpha_frequency + float(count > 0) * alpha_presence; + } + + candidates->sorted = false; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * candidates, const struct llama_grammar * grammar) { + assert(ctx); + const int64_t t_start_sample_us = ggml_time_us(); + + bool allow_eos = false; + for (const auto & stack : grammar->stacks) { + if (stack.empty()) { + allow_eos = true; + break; + } + } + + const llama_token eos = llama_token_eos(); + + std::vector> candidates_decoded; + std::vector candidates_grammar; + + for (size_t i = 0; i < candidates->size; ++i) { + const llama_token id = candidates->data[i].id; + const char * str = llama_token_to_str(ctx, id); + if (id == eos) { + if (!allow_eos) { + candidates->data[i].logit = -INFINITY; + } + } else if (*str == 0) { + candidates->data[i].logit = -INFINITY; + } else { + candidates_decoded.push_back(decode_utf8(str)); + candidates_grammar.push_back({ i, candidates_decoded.back().data() }); + } + } + + const auto rejects = + llama_grammar_reject_candidates(grammar->rules, grammar->stacks, candidates_grammar); + for (auto & reject : rejects) { + candidates->data[reject.index].logit = -INFINITY; + } + + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; +} + +static void llama_log_softmax(float * array, size_t size) { + float max_l = *std::max_element(array, array + size); + float sum = 0.f; + for (size_t i = 0; i < size; ++i) { + float p = expf(array[i] - max_l); + sum += p; + array[i] = p; + } + + for (size_t i = 0; i < size; ++i) { + array[i] = logf(array[i] / sum); + } +} + +void llama_sample_classifier_free_guidance( + struct llama_context * ctx, + llama_token_data_array * candidates, + struct llama_context * guidance_ctx, + float scale) { + int64_t t_start_sample_us = ggml_time_us(); + + assert(ctx); + auto n_vocab = llama_n_vocab(ctx); + assert(n_vocab == (int)candidates->size); + assert(!candidates->sorted); + + std::vector logits_base; + logits_base.reserve(candidates->size); + for (size_t i = 0; i < candidates->size; ++i) { + logits_base.push_back(candidates->data[i].logit); + } + llama_log_softmax(logits_base.data(), candidates->size); + + float* logits_guidance = llama_get_logits(guidance_ctx); + llama_log_softmax(logits_guidance, n_vocab); + + for (int i = 0; i < n_vocab; ++i) { + float logit_guidance = logits_guidance[i]; + float logit_base = logits_base[i]; + candidates->data[i].logit = scale * (logit_base - logit_guidance) + logit_guidance; + } + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int m, float * mu) { + assert(ctx); + auto N = float(llama_n_vocab(ctx)); + int64_t t_start_sample_us; + t_start_sample_us = ggml_time_us(); + + llama_sample_softmax(nullptr, candidates); + + // Estimate s_hat using the most probable m tokens + float s_hat = 0.0; + float sum_ti_bi = 0.0; + float sum_ti_sq = 0.0; + for (size_t i = 0; i < size_t(m - 1) && i < candidates->size - 1; ++i) { + float t_i = logf(float(i + 2) / float(i + 1)); + float b_i = logf(candidates->data[i].p / candidates->data[i + 1].p); + sum_ti_bi += t_i * b_i; + sum_ti_sq += t_i * t_i; + } + s_hat = sum_ti_bi / sum_ti_sq; + + // Compute k from the estimated s_hat and target surprise value + float epsilon_hat = s_hat - 1; + float k = powf((epsilon_hat * powf(2, *mu)) / (1 - powf(N, -epsilon_hat)), 1 / s_hat); + + // Sample the next word X using top-k sampling + llama_sample_top_k(nullptr, candidates, int(k), 1); + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } + llama_token X = llama_sample_token(ctx, candidates); + t_start_sample_us = ggml_time_us(); + + // Compute error as the difference between observed surprise and target surprise value + size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) { + return candidate.id == X; + })); + float observed_surprise = -log2f(candidates->data[X_idx].p); + float e = observed_surprise - tau; + + // Update mu using the learning rate and error + *mu = *mu - eta * e; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } + return X; +} + +llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu) { + int64_t t_start_sample_us; + t_start_sample_us = ggml_time_us(); + + llama_sample_softmax(ctx, candidates); + + // Truncate the words with surprise values greater than mu + candidates->size = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) { + return -log2f(candidate.p) > *mu; + })); + + if (candidates->size == 0) { + candidates->size = 1; + } + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } + + // Normalize the probabilities of the remaining words + llama_sample_softmax(ctx, candidates); + + // Sample the next word X from the remaining words + llama_token X = llama_sample_token(ctx, candidates); + t_start_sample_us = ggml_time_us(); + + // Compute error as the difference between observed surprise and target surprise value + size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) { + return candidate.id == X; + })); + float observed_surprise = -log2f(candidates->data[X_idx].p); + float e = observed_surprise - tau; + + // Update mu using the learning rate and error + *mu = *mu - eta * e; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } + return X; +} + +llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates) { + const int64_t t_start_sample_us = ggml_time_us(); + + // Find max element + auto * max_iter = std::max_element(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) { + return a.logit < b.logit; + }); + + llama_token result = max_iter->id; + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + ctx->n_sample++; + } + return result; +} + +llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates) { + assert(ctx); + const int64_t t_start_sample_us = ggml_time_us(); + llama_sample_softmax(nullptr, candidates); + + std::vector probs; + probs.reserve(candidates->size); + for (size_t i = 0; i < candidates->size; ++i) { + probs.push_back(candidates->data[i].p); + } + + std::discrete_distribution<> dist(probs.begin(), probs.end()); + auto & rng = ctx->rng; + int idx = dist(rng); + + llama_token result = candidates->data[idx].id; + + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + ctx->n_sample++; + return result; +} + +void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar * grammar, llama_token token) { + const int64_t t_start_sample_us = ggml_time_us(); + + if (token == llama_token_eos()) { + for (const auto & stack : grammar->stacks) { + if (stack.empty()) { + return; + } + } + GGML_ASSERT(false); + } + + const char * str = llama_token_to_str(ctx, token); + // Note terminating 0 in decoded string + auto code_points = decode_utf8(str); + for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { + grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); + } + GGML_ASSERT(!grammar->stacks.empty()); + + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; +} + +// +// quantization +// + +static void llama_convert_tensor_internal(const llama_load_tensor & tensor, gguf_buffer & output, const int nelements, const int nthread) { + if (output.size < nelements * sizeof(float)) { + output.resize(nelements * sizeof(float)); + } + float * f32_output = (float *) output.addr; + + ggml_type_traits_t qtype; + if (ggml_is_quantized(tensor.type)) { + qtype = ggml_internal_get_type_traits(tensor.type); + if (qtype.to_float == NULL) { + throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor.type))); + } + } else if (tensor.type != GGML_TYPE_F16) { + throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor.type))); + } + + if (nthread < 2) { + if (tensor.type == GGML_TYPE_F16) { + ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor.data, f32_output, nelements); + } else if (ggml_is_quantized(tensor.type)) { + qtype.to_float(tensor.data, f32_output, nelements); + } else { + GGML_ASSERT(false); // unreachable + } + return; + } + + auto block_size = tensor.type == GGML_TYPE_F16 ? 1 : (size_t)ggml_blck_size(tensor.type); + auto block_size_bytes = ggml_type_size(tensor.type); + + GGML_ASSERT(nelements % block_size == 0); + auto nblocks = nelements / block_size; + auto blocks_per_thread = nblocks / nthread; + auto spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count + + std::vector workers; + for (auto tnum = 0, in_buff_offs = 0, out_buff_offs = 0; tnum < nthread; tnum++) { + auto thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread + auto thr_elems = thr_blocks * block_size; // number of elements for this thread + auto thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread + + auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) { + if (typ == GGML_TYPE_F16) { + ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels); + } else { + qtype.to_float(inbuf, outbuf, nels); + } + }; + workers.push_back(std::thread(compute, tensor.type, tensor.data + in_buff_offs, f32_output + out_buff_offs, thr_elems)); + in_buff_offs += thr_block_bytes; + out_buff_offs += thr_elems; + } + for (auto & worker : workers) { + worker.join(); + } + +} + +static void llama_model_quantize_internal(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) { + ggml_type quantized_type; + llama_ftype ftype = params->ftype; + int nthread = params->nthread; + + switch (params->ftype) { + case LLAMA_FTYPE_MOSTLY_Q4_0: quantized_type = GGML_TYPE_Q4_0; break; + case LLAMA_FTYPE_MOSTLY_Q4_1: quantized_type = GGML_TYPE_Q4_1; break; + case LLAMA_FTYPE_MOSTLY_Q5_0: quantized_type = GGML_TYPE_Q5_0; break; + case LLAMA_FTYPE_MOSTLY_Q5_1: quantized_type = GGML_TYPE_Q5_1; break; + case LLAMA_FTYPE_MOSTLY_Q8_0: quantized_type = GGML_TYPE_Q8_0; break; + case LLAMA_FTYPE_MOSTLY_F16: quantized_type = GGML_TYPE_F16; break; + case LLAMA_FTYPE_ALL_F32: quantized_type = GGML_TYPE_F32; break; + +#ifdef GGML_USE_K_QUANTS + // K-quants + case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break; + case LLAMA_FTYPE_MOSTLY_Q3_K_S: + case LLAMA_FTYPE_MOSTLY_Q3_K_M: + case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break; + case LLAMA_FTYPE_MOSTLY_Q4_K_S: + case LLAMA_FTYPE_MOSTLY_Q4_K_M: quantized_type = GGML_TYPE_Q4_K; break; + case LLAMA_FTYPE_MOSTLY_Q5_K_S: + case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break; + case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break; +#endif + default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); + } + + if (nthread <= 0) { + nthread = std::thread::hardware_concurrency(); + } + + std::unique_ptr model_loader(new llama_model_loader(fname_inp, /*use_mmap*/ false)); + gguf_file_saver file_saver(fname_out.c_str(), model_loader->file_loader.get(), params->ftype); + +#ifdef GGML_USE_K_QUANTS + int n_attention_wv = 0; + int n_feed_forward_w2 = 0; + for (auto& tensor : model_loader->tensors_map.tensors) { + if (tensor.name.find("attention.wv.weight") != std::string::npos) { + ++n_attention_wv; + } + else if (tensor.name.find("feed_forward.w2.weight") != std::string::npos) { + ++n_feed_forward_w2; + } + } + + int i_attention_wv = 0; + int i_feed_forward_w2 = 0; +#endif + + size_t total_size_org = 0; + size_t total_size_new = 0; + std::vector hist_all(1 << 4, 0); + + std::vector workers; + std::mutex mutex; + + auto use_more_bits = [] (int i_layer, int num_layers) -> bool { + return i_layer < num_layers/8 || i_layer >= 7*num_layers/8 || (i_layer - num_layers/8)%3 == 2; + }; + + size_t idx = 0; + for (llama_load_tensor & tensor : model_loader->tensors_map.tensors) { + gguf_buffer read_data; + read_data.resize(tensor.size); + tensor.data = read_data.addr; + model_loader->load_data_for(tensor); + + printf("[%4zu/%4zu] %36s - %16s, type = %6s, ", + ++idx, model_loader->tensors_map.tensors.size(), + tensor.name.c_str(), llama_format_tensor_shape(tensor.ne).c_str(), + ggml_type_name(tensor.type)); + + // This used to be a regex, but has an extreme cost to compile times. + bool quantize = tensor.name.rfind("weight") == tensor.name.size() - 6; // ends with 'weight'? + + // quantize only 2D tensors + quantize &= (tensor.ne.size() == 2); + quantize &= params->quantize_output_tensor || tensor.name != "output.weight"; + quantize &= quantized_type != tensor.type; + + enum ggml_type new_type; + void * new_data; + size_t new_size; + gguf_buffer work; + + if (!quantize) { + new_type = tensor.type; + new_data = tensor.data; + new_size = tensor.size; + printf("size = %8.3f MB\n", tensor.size/1024.0/1024.0); + } else { + new_type = quantized_type; +#ifdef GGML_USE_K_QUANTS + if (tensor.name == "output.weight") { + int nx = tensor.ne.at(0); + int ny = tensor.ne.at(1); + if (nx % QK_K == 0 && ny % QK_K == 0) { + new_type = GGML_TYPE_Q6_K; + } + } else if (tensor.name.find("attention.wv.weight") != std::string::npos) { + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; + else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) && + use_more_bits(i_attention_wv, n_attention_wv)) new_type = GGML_TYPE_Q6_K; + else if (QK_K == 64 && (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S) && + (i_attention_wv < n_attention_wv/8 || i_attention_wv >= 7*n_attention_wv/8)) new_type = GGML_TYPE_Q6_K; + ++i_attention_wv; + } else if (tensor.name.find("feed_forward.w2.weight") != std::string::npos) { + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; + else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) && + use_more_bits(i_feed_forward_w2, n_feed_forward_w2)) new_type = GGML_TYPE_Q6_K; + //else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && i_feed_forward_w2 < n_feed_forward_w2/8) new_type = GGML_TYPE_Q6_K; + ++i_feed_forward_w2; + } else if (tensor.name.find("attention.wo.weight") != std::string::npos) { + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; + } + bool convert_incompatible_tensor = false; + if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K || + new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K) { + int nx = tensor.ne.at(0); + int ny = tensor.ne.at(1); + if (nx % QK_K != 0 || ny % QK_K != 0) { + fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K); + convert_incompatible_tensor = true; + } + } + if (convert_incompatible_tensor) { + if (tensor.name == "output.weight") { + new_type = GGML_TYPE_F16; //fall back to F16 instead of just failing. + fprintf(stderr, "F16 will be used for this tensor instead.\n"); + } else if (tensor.name == "tok_embeddings.weight") { + new_type = GGML_TYPE_Q4_0; //fall back to Q4_0 instead of just failing. + fprintf(stderr, "Q4_0 will be used for this tensor instead.\n"); + } else { + throw std::runtime_error("Unsupported tensor size encountered\n"); + } + } +#endif + + float * f32_data; + size_t nelements = tensor.ne.at(0) * tensor.ne.at(1); + gguf_buffer f32_conv_buf; + + if (tensor.type == GGML_TYPE_F32) { + f32_data = (float *) tensor.data; + } else if (ggml_is_quantized(tensor.type) && !params->allow_requantize) { + throw std::runtime_error(format("requantizing from type %s is disabled", ggml_type_name(tensor.type))); + } else { + llama_convert_tensor_internal(tensor, f32_conv_buf, nelements, nthread); + f32_data = (float *) f32_conv_buf.addr; + } + + printf("quantizing to %s .. ", ggml_type_name(new_type)); + fflush(stdout); + + work.resize(nelements * 4); // upper bound on size + new_data = work.addr; + std::vector hist_cur(1 << 4, 0); + + int chunk_size = 32 * 512; + const int nchunk = (nelements + chunk_size - 1)/chunk_size; + const int nthread_use = nthread > 1 ? std::max(1, std::min(nthread, nchunk)) : 1; + if (nthread_use < 2) { + new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nelements, hist_cur.data()); + } else { + size_t counter = 0; + new_size = 0; + auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, nelements, chunk_size] () { + std::vector local_hist; + size_t local_size = 0; + while (true) { + std::unique_lock lock(mutex); + size_t first = counter; counter += chunk_size; + if (first >= nelements) { + if (!local_hist.empty()) { + for (int j=0; j %8.2f MB | hist: ", tensor.size/1024.0/1024.0, new_size/1024.0/1024.0); + int64_t tot_count = 0; + for (size_t i = 0; i < hist_cur.size(); i++) { + hist_all[i] += hist_cur[i]; + tot_count += hist_cur[i]; + } + + if (tot_count > 0) { + for (size_t i = 0; i < hist_cur.size(); i++) { + printf("%5.3f ", hist_cur[i] / float(nelements)); + } + } + printf("\n"); + } + total_size_org += tensor.size; + total_size_new += new_size; + file_saver.write_tensor(tensor, new_type, new_data, new_size); + } + + printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0); + printf("%s: quant size = %8.2f MB\n", __func__, total_size_new/1024.0/1024.0); + + { + int64_t sum_all = 0; + for (size_t i = 0; i < hist_all.size(); i++) { + sum_all += hist_all[i]; + } + + if (sum_all > 0) { + printf("%s: hist: ", __func__); + for (size_t i = 0; i < hist_all.size(); i++) { + printf("%5.3f ", hist_all[i] / float(sum_all)); + } + printf("\n"); + } + } +} + + + +// +// interface implementation +// + +struct llama_model * llama_load_model_from_file( + const char * path_model, + struct llama_context_params params) { + ggml_time_init(); + + llama_model * model = new llama_model; + + ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32; + + if (!llama_model_load(path_model, *model, model->vocab, params.n_ctx, params.n_batch, params.n_gqa, params.rms_norm_eps, params.n_gpu_layers, + params.main_gpu, params.tensor_split, params.rope_freq_base, params.rope_freq_scale,params.low_vram, + memory_type, params.use_mmap, params.use_mlock, params.vocab_only, params.progress_callback, + params.progress_callback_user_data)) { + delete model; + fprintf(stderr, "%s: failed to load model\n", __func__); + return nullptr; + } + + return model; +} + +void llama_free_model(struct llama_model * model) { + delete model; +} + +struct llama_context * llama_new_context_with_model( + struct llama_model * model, + struct llama_context_params params) { + + if (!model) { + return nullptr; + } + + llama_context * ctx = new llama_context(*model); + + if (params.seed == LLAMA_DEFAULT_SEED) { + params.seed = time(NULL); + } + + unsigned cur_percentage = 0; + if (params.progress_callback == NULL) { + params.progress_callback_user_data = &cur_percentage; + params.progress_callback = [](float progress, void * ctx) { + unsigned * cur_percentage_p = (unsigned *) ctx; + unsigned percentage = (unsigned) (100 * progress); + while (percentage > *cur_percentage_p) { + *cur_percentage_p = percentage; + fprintf(stderr, "."); + fflush(stderr); + if (percentage >= 100) { + fprintf(stderr, "\n"); + } + } + }; + } + + ctx->rng = std::mt19937(params.seed); + ctx->logits_all = params.logits_all; + + ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32; + + // reserve memory for context buffers + if (!params.vocab_only) { + if (!kv_cache_init(ctx->model.hparams, ctx->kv_self, memory_type, ctx->model.hparams.n_ctx, params.n_gpu_layers)) { + fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__); + llama_free(ctx); + return nullptr; + } + + { + const size_t memory_size = ggml_nbytes(ctx->kv_self.k) + ggml_nbytes(ctx->kv_self.v); + fprintf(stderr, "%s: kv self size = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0); + } + + const auto & hparams = ctx->model.hparams; + + // resized during inference + if (params.logits_all) { + ctx->logits.reserve(hparams.n_ctx*hparams.n_vocab); + } else { + ctx->logits.reserve(hparams.n_vocab); + } + + if (params.embedding){ + ctx->embedding.resize(hparams.n_embd); + } + + ctx->buf_compute.resize(MEM_REQ_EVAL().at(ctx->model.type) + ggml_graph_overhead()); + + ctx->buf_scratch[0].resize(MEM_REQ_SCRATCH0(hparams.n_ctx).at(ctx->model.type)); + ctx->buf_scratch[1].resize(MEM_REQ_SCRATCH1().at(ctx->model.type)); + } + +#ifdef GGML_USE_METAL + if (params.n_gpu_layers > 0) { + // this allocates all Metal resources and memory buffers + ctx->ctx_metal = ggml_metal_init(1); + + void * data_ptr = NULL; + size_t data_size = 0; + + if (params.use_mmap) { + data_ptr = ctx->model.mapping->addr; + data_size = ctx->model.mapping->size; + } else { + data_ptr = ggml_get_mem_buffer(ctx->model.ctx); + data_size = ggml_get_mem_size (ctx->model.ctx); + } + + const size_t max_size = ggml_get_max_tensor_size(ctx->model.ctx); + + fprintf(stderr, "%s: max tensor size = %8.2f MB\n", __func__, max_size/1024.0/1024.0); + +#define LLAMA_METAL_CHECK_BUF(result) \ + if (!(result)) { \ + fprintf(stderr, "%s: failed to add buffer\n", __func__); \ + llama_free(ctx); \ + return NULL; \ + } + + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "data", data_ptr, data_size, max_size)); + + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "eval", ctx->buf_compute.addr, ctx->buf_compute.size, 0)); + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "kv", ctx->kv_self.buf.addr, ctx->kv_self.buf.size, 0)); + + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr0", ctx->buf_scratch[0].addr, ctx->buf_scratch[0].size, 0)); + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr1", ctx->buf_scratch[1].addr, ctx->buf_scratch[1].size, 0)); +#undef LLAMA_METAL_CHECK_BUF + } +#endif + +#ifdef GGML_USE_MPI + ctx->ctx_mpi = ggml_mpi_init(); + + if (ggml_mpi_rank(ctx->ctx_mpi) > 0) { + // Enter a blocking eval loop with dummy input, letting rank=0 drive the process + const std::vector tmp(ctx->model.hparams.n_ctx, llama_token_bos()); + while (!llama_eval(ctx, tmp.data(), tmp.size(), 0, 0)) {}; + llama_backend_free(); + exit(1); + } +#endif + + return ctx; +} + +struct llama_context * llama_init_from_file( + const char * path_model, + struct llama_context_params params) { + + struct llama_model * model = llama_load_model_from_file(path_model, params); + if (!model) { + return nullptr; + } + struct llama_context * ctx = llama_new_context_with_model(model, params); + ctx->model_owner = true; + return ctx; +} + +void llama_free(struct llama_context * ctx) { + if (ctx->model_owner) { + delete &ctx->model; + } + delete ctx; +} + +int llama_model_quantize( + const char * fname_inp, + const char * fname_out, + const llama_model_quantize_params *params) { + try { + llama_model_quantize_internal(fname_inp, fname_out, params); + return 0; + } catch (const std::exception & err) { + fprintf(stderr, "%s: failed to quantize: %s\n", __func__, err.what()); + return 1; + } +} + +int llama_apply_lora_from_file_internal(const struct llama_model & model, const char * path_lora, const char * path_base_model, int n_threads) { + fprintf(stderr, "%s: applying lora adapter from '%s' - please wait ...\n", __func__, path_lora); + + const int64_t t_start_lora_us = ggml_time_us(); + + auto fin = std::ifstream(path_lora, std::ios::binary); + if (!fin) { + fprintf(stderr, "%s: failed to open '%s'\n", __func__, path_lora); + return 1; + } + + // verify magic and version + { + uint32_t magic; + fin.read((char *) &magic, sizeof(magic)); + if (magic != LLAMA_FILE_MAGIC_GGLA) { + fprintf(stderr, "%s: bad file magic\n", __func__); + return 1; + } + uint32_t format_version; + fin.read((char *) &format_version, sizeof(format_version)); + + if (format_version != 1) { + fprintf(stderr, "%s: unsupported file version\n", __func__ ); + return 1; + } + } + + int32_t lora_r; + int32_t lora_alpha; + fin.read((char *) &lora_r, sizeof(lora_r)); + fin.read((char *) &lora_alpha, sizeof(lora_alpha)); + float scaling = (float)lora_alpha / (float)lora_r; + + fprintf(stderr, "%s: r = %d, alpha = %d, scaling = %.2f\n", __func__, lora_r, lora_alpha, scaling); + + + // create a temporary ggml context to store the lora tensors + // todo: calculate size from biggest possible tensor + std::vector lora_buf(1024ull * 1024ull * 1024ull); + struct ggml_init_params params; + params.mem_size = lora_buf.size(); + params.mem_buffer = lora_buf.data(); + params.no_alloc = false; + + ggml_context * lora_ctx = ggml_init(params); + std::unordered_map lora_tensors; + + // create a name -> tensor map of the model to accelerate lookups + std::unordered_map model_tensors; + for (const auto & kv: model.tensors_by_name) { + model_tensors.insert(kv); + } + + + // load base model + std::unique_ptr model_loader; + ggml_context * base_ctx = NULL; + gguf_buffer base_buf; + if (path_base_model) { + fprintf(stderr, "%s: loading base model from '%s'\n", __func__, path_base_model); + model_loader.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true)); + + size_t ctx_size; + size_t mmapped_size; + model_loader->calc_sizes(&ctx_size, &mmapped_size); + base_buf.resize(ctx_size); + + ggml_init_params base_params; + base_params.mem_size = base_buf.size; + base_params.mem_buffer = base_buf.addr; + base_params.no_alloc = model_loader->use_mmap; + + base_ctx = ggml_init(base_params); + + model_loader->ggml_ctx = base_ctx; + + // maybe this should in llama_model_loader + if (model_loader->use_mmap) { + model_loader->mapping.reset(new gguf_mmap(&model_loader->file_loader->file, /* prefetch */ 0, ggml_is_numa())); + } + } + + // read tensors and apply + bool warned = false; + int n_tensors = 0; + + std::vector work_buffer; + + while (true) { + int32_t n_dims; + int32_t length; + int32_t ftype; + + fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); + fin.read(reinterpret_cast(&length), sizeof(length)); + fin.read(reinterpret_cast(&ftype), sizeof(ftype)); + if (fin.eof()) { + break; + } + + int32_t ne[2] = { 1, 1 }; + for (int i = 0; i < n_dims; ++i) { + fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i])); + } + + std::string name; + { + char buf[1024]; + fin.read(buf, length); + name = std::string(buf, length); + } + + // check for lora suffix and get the type of tensor + const std::string lora_suffix = ".lora"; + size_t pos = name.rfind(lora_suffix); + if (pos == std::string::npos) { + fprintf(stderr, "%s: error: '%s' is not a lora tensor\n", __func__, name.c_str()); + return 1; + } + + std::string lora_type = name.substr(pos + lora_suffix.length()); + std::string base_name = name; + base_name.erase(pos); + // fprintf(stderr, "%s: %s => %s (lora type %s) ", __func__, name.c_str(),base_name.c_str(), lora_type.c_str()); + + if (model_tensors.find(base_name) == model_tensors.end()) { + fprintf(stderr, "%s: unknown tensor '%s' in lora adapter\n", __func__, name.data()); + return 1; + } + + // create ggml tensor + ggml_type wtype; + switch (ftype) { + case 0: wtype = GGML_TYPE_F32; break; + case 1: wtype = GGML_TYPE_F16; break; + default: + { + fprintf(stderr, "%s: invalid tensor data type '%d'\n", + __func__, ftype); + return false; + } + } + ggml_tensor * lora_tensor; + if (n_dims == 2) { + lora_tensor = ggml_new_tensor_2d(lora_ctx, wtype, ne[0], ne[1]); + } + else { + fprintf(stderr, "%s: unsupported tensor dimension %d\n", __func__, n_dims); + return 1; + } + ggml_set_name(lora_tensor, "lora_tensor"); + + // load tensor data + size_t offset = fin.tellg(); + size_t tensor_data_size = ggml_nbytes(lora_tensor); + offset = (offset + 31) & -32; + fin.seekg(offset); + fin.read((char*)lora_tensor->data, tensor_data_size); + + lora_tensors[name] = lora_tensor; + + // check if we have both A and B tensors and apply + if (lora_tensors.find(base_name + ".loraA") != lora_tensors.end() && + lora_tensors.find(base_name + ".loraB") != lora_tensors.end()) { + + ggml_tensor * dest_t = model_tensors[base_name]; + + offload_func_t offload_func = llama_nop; + offload_func_t offload_func_force_inplace = llama_nop; + +#ifdef GGML_USE_CUBLAS + if (dest_t->backend == GGML_BACKEND_GPU || dest_t->backend == GGML_BACKEND_GPU_SPLIT) { + if (dest_t->type != GGML_TYPE_F16) { + throw std::runtime_error(format( + "%s: error: the simultaneous use of LoRAs and GPU acceleration is only supported for f16 models", __func__)); + } + offload_func = ggml_cuda_assign_buffers; + offload_func_force_inplace = ggml_cuda_assign_buffers_force_inplace; + } +#endif // GGML_USE_CUBLAS + + ggml_tensor * base_t; + if (model_loader) { + // load from base model + if (model_loader->tensors_map.name_to_idx.find(base_name) == model_loader->tensors_map.name_to_idx.end()) { + fprintf(stderr, "%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str()); + return 1; + } + size_t idx = model_loader->tensors_map.name_to_idx[base_name]; + llama_load_tensor & lt = model_loader->tensors_map.tensors[idx]; + base_t = model_loader->get_tensor(base_name, { (uint32_t)dest_t->ne[0], (uint32_t)dest_t->ne[1] }, GGML_BACKEND_CPU); + lt.data = (uint8_t *) lt.ggml_tensor->data; + model_loader->load_data_for(lt); + lt.ggml_tensor->data = lt.data; + } + else { + base_t = dest_t; + } + + if (ggml_is_quantized(base_t->type)) { + if (!warned) { + fprintf(stderr, "%s: warning: using a lora adapter with a quantized model may result in poor quality, " + "use a f16 or f32 base model with --lora-base\n", __func__); + warned = true; + } + } + + ggml_tensor * loraA = lora_tensors[base_name + ".loraA"]; + GGML_ASSERT(loraA->type == GGML_TYPE_F32); + ggml_set_name(loraA, "loraA"); + + ggml_tensor * loraB = lora_tensors[base_name + ".loraB"]; + GGML_ASSERT(loraB->type == GGML_TYPE_F32); + ggml_set_name(loraB, "loraB"); + + if (base_t->ne[0] != loraA->ne[1] || base_t->ne[1] != loraB->ne[1]) { + fprintf(stderr, "%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");" + " are you sure that this adapter is for this model?\n", __func__, base_t->ne[0], loraA->ne[1]); + return 1; + } + + // w = w + BA*s + ggml_tensor * BA = ggml_mul_mat(lora_ctx, loraA, loraB); + offload_func(BA); + ggml_set_name(BA, "BA"); + + if (scaling != 1.0f) { + ggml_tensor * scale_tensor = ggml_new_f32(lora_ctx, scaling); + ggml_set_name(scale_tensor, "scale_tensor"); + + BA = ggml_scale_inplace(lora_ctx, BA, scale_tensor); + offload_func(BA); + ggml_set_name(BA, "BA_scaled"); + } + + ggml_tensor * r; + if (base_t == dest_t) { + r = ggml_add_inplace(lora_ctx, dest_t, BA); + offload_func_force_inplace(r); + ggml_set_name(r, "r_add_inplace"); + } + else { + r = ggml_add(lora_ctx, base_t, BA); + offload_func(r); + ggml_set_name(r, "r_add"); + + r = ggml_cpy(lora_ctx, r, dest_t); + offload_func(r); + ggml_set_name(r, "r_cpy"); + } + + struct ggml_cgraph gf = ggml_build_forward(r); + + ggml_graph_compute_helper(work_buffer, &gf, n_threads); + + // we won't need these tensors again, reset the context to save memory + ggml_free(lora_ctx); + lora_ctx = ggml_init(params); + lora_tensors.clear(); + + n_tensors++; + if (n_tensors % 4 == 0) { + fprintf(stderr, "."); + } + } + } + + // TODO: this should be in a destructor, it will leak on failure + ggml_free(lora_ctx); + if (base_ctx) { + ggml_free(base_ctx); + } + + const int64_t t_lora_us = ggml_time_us() - t_start_lora_us; + fprintf(stderr, " done (%.2f ms)\n", t_lora_us / 1000.0); + + return 0; +} + +int llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lora, const char * path_base_model, int n_threads) { + try { + return llama_apply_lora_from_file_internal(ctx->model, path_lora, path_base_model, n_threads); + } catch (const std::exception & err) { + fprintf(stderr, "%s: failed to apply lora adapter: %s\n", __func__, err.what()); + return 1; + } +} + +int llama_model_apply_lora_from_file(const struct llama_model * model, const char * path_lora, const char * path_base_model, int n_threads) { + try { + return llama_apply_lora_from_file_internal(*model, path_lora, path_base_model, n_threads); + } catch (const std::exception & err) { + fprintf(stderr, "%s: failed to apply lora adapter: %s\n", __func__, err.what()); + return 1; + } +} + +int llama_get_kv_cache_token_count(const struct llama_context * ctx) { + return ctx->kv_self.n; +} + +#define LLAMA_MAX_RNG_STATE (64*1024) + +void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed) { + if (seed == LLAMA_DEFAULT_SEED) { + seed = time(NULL); + } + ctx->rng.seed(seed); +} + +// Returns the *maximum* size of the state +size_t llama_get_state_size(const struct llama_context * ctx) { + // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state. + // for reference, std::mt19937(1337) serializes to 6701 bytes. + const size_t s_rng_size = sizeof(size_t); + const size_t s_rng = LLAMA_MAX_RNG_STATE; + const size_t s_logits_capacity = sizeof(size_t); + const size_t s_logits_size = sizeof(size_t); + const size_t s_logits = ctx->logits.capacity() * sizeof(float); + const size_t s_embedding_size = sizeof(size_t); + const size_t s_embedding = ctx->embedding.size() * sizeof(float); + const size_t s_kv_size = sizeof(size_t); + const size_t s_kv_ntok = sizeof(int); + const size_t s_kv = ctx->kv_self.buf.size; + + const size_t s_total = ( + + s_rng_size + + s_rng + + s_logits_capacity + + s_logits_size + + s_logits + + s_embedding_size + + s_embedding + + s_kv_size + + s_kv_ntok + + s_kv + ); + + return s_total; +} + +// Copies the state to the specified destination address +size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { + uint8_t * out = dst; + + // copy rng + { + std::stringstream rng_ss; + rng_ss << ctx->rng; + + const size_t rng_size = rng_ss.str().size(); + char rng_buf[LLAMA_MAX_RNG_STATE]; + + memset(&rng_buf[0], 0, LLAMA_MAX_RNG_STATE); + memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size()); + + memcpy(out, &rng_size, sizeof(rng_size)); out += sizeof(rng_size); + memcpy(out, &rng_buf[0], LLAMA_MAX_RNG_STATE); out += LLAMA_MAX_RNG_STATE; + } + + // copy logits + { + const size_t logits_cap = ctx->logits.capacity(); + const size_t logits_size = ctx->logits.size(); + + memcpy(out, &logits_cap, sizeof(logits_cap)); out += sizeof(logits_cap); + memcpy(out, &logits_size, sizeof(logits_size)); out += sizeof(logits_size); + + if (logits_size) { + memcpy(out, ctx->logits.data(), logits_size * sizeof(float)); + } + + out += logits_cap * sizeof(float); + } + + // copy embeddings + { + const size_t embedding_size = ctx->embedding.size(); + + memcpy(out, &embedding_size, sizeof(embedding_size)); out += sizeof(embedding_size); + + if (embedding_size) { + memcpy(out, ctx->embedding.data(), embedding_size * sizeof(float)); + out += embedding_size * sizeof(float); + } + } + + // copy kv cache + { + const auto & kv_self = ctx->kv_self; + const auto & hparams = ctx->model.hparams; + const int n_layer = hparams.n_layer; + const int n_embd = hparams.n_embd; + const int n_ctx = hparams.n_ctx; + + const size_t kv_size = kv_self.buf.size; + const int kv_ntok = llama_get_kv_cache_token_count(ctx); + + memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size); + memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok); + + if (kv_size) { + const size_t elt_size = ggml_element_size(kv_self.k); + + ggml_context * cpy_ctx = ggml_init({ 4096, NULL, /* no_alloc */ true }); + ggml_cgraph gf{}; + + ggml_tensor * kout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_ntok, n_layer); + kout3d->data = out; + out += ggml_nbytes(kout3d); + + ggml_tensor * vout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_ntok, n_embd, n_layer); + vout3d->data = out; + out += ggml_nbytes(vout3d); + + ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k, + n_embd, kv_ntok, n_layer, + elt_size*n_embd, elt_size*n_embd*n_ctx, 0); + + ggml_tensor * v3d = ggml_view_3d(cpy_ctx, kv_self.v, + kv_ntok, n_embd, n_layer, + elt_size*n_ctx, elt_size*n_ctx*n_embd, 0); + + ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, k3d, kout3d)); + ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, v3d, vout3d)); + ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1); + + ggml_free(cpy_ctx); + } + } + + const size_t written = out - dst; + const size_t max_size = llama_get_state_size(ctx); + + GGML_ASSERT(written <= max_size); + + return written; +} + +// Sets the state reading from the specified source address +size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { + uint8_t * inp = src; + + // set rng + { + size_t rng_size; + char rng_buf[LLAMA_MAX_RNG_STATE]; + + memcpy(&rng_size, inp, sizeof(rng_size)); inp += sizeof(rng_size); + memcpy(&rng_buf[0], inp, LLAMA_MAX_RNG_STATE); inp += LLAMA_MAX_RNG_STATE; + + std::stringstream rng_ss; + rng_ss.str(std::string(&rng_buf[0], rng_size)); + rng_ss >> ctx->rng; + + GGML_ASSERT(rng_ss.fail() == false); + } + + // set logits + { + size_t logits_cap; + size_t logits_size; + + memcpy(&logits_cap, inp, sizeof(logits_cap)); inp += sizeof(logits_cap); + memcpy(&logits_size, inp, sizeof(logits_size)); inp += sizeof(logits_size); + + GGML_ASSERT(ctx->logits.capacity() == logits_cap); + + if (logits_size) { + ctx->logits.resize(logits_size); + memcpy(ctx->logits.data(), inp, logits_size * sizeof(float)); + } + + inp += logits_cap * sizeof(float); + } + + // set embeddings + { + size_t embedding_size; + + memcpy(&embedding_size, inp, sizeof(embedding_size)); inp += sizeof(embedding_size); + + GGML_ASSERT(ctx->embedding.capacity() == embedding_size); + + if (embedding_size) { + memcpy(ctx->embedding.data(), inp, embedding_size * sizeof(float)); + inp += embedding_size * sizeof(float); + } + } + + // set kv cache + { + const auto & kv_self = ctx->kv_self; + const auto & hparams = ctx->model.hparams; + const int n_layer = hparams.n_layer; + const int n_embd = hparams.n_embd; + const int n_ctx = hparams.n_ctx; + + size_t kv_size; + int kv_ntok; + + memcpy(&kv_size, inp, sizeof(kv_size)); inp += sizeof(kv_size); + memcpy(&kv_ntok, inp, sizeof(kv_ntok)); inp += sizeof(kv_ntok); + + if (kv_size) { + GGML_ASSERT(kv_self.buf.size == kv_size); + + const size_t elt_size = ggml_element_size(kv_self.k); + + ggml_context * cpy_ctx = ggml_init({ 4096, NULL, /* no_alloc */ true }); + ggml_cgraph gf{}; + + ggml_tensor * kin3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_ntok, n_layer); + kin3d->data = (void *) inp; + inp += ggml_nbytes(kin3d); + + ggml_tensor * vin3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_ntok, n_embd, n_layer); + vin3d->data = (void *) inp; + inp += ggml_nbytes(vin3d); + + ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k, + n_embd, kv_ntok, n_layer, + elt_size*n_embd, elt_size*n_embd*n_ctx, 0); + + ggml_tensor * v3d = ggml_view_3d(cpy_ctx, kv_self.v, + kv_ntok, n_embd, n_layer, + elt_size*n_ctx, elt_size*n_ctx*n_embd, 0); + + ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, kin3d, k3d)); + ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, vin3d, v3d)); + ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1); + + ggml_free(cpy_ctx); + } + + ctx->kv_self.n = kv_ntok; + } + + const size_t nread = inp - src; + const size_t max_size = llama_get_state_size(ctx); + + GGML_ASSERT(nread <= max_size); + + return nread; +} + +static bool llama_load_session_file_internal(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { + gguf_file file(path_session, "rb"); + GGML_UNUSED(ctx); + GGML_UNUSED(path_session); + GGML_UNUSED(tokens_out); + GGML_UNUSED(n_token_capacity); + GGML_UNUSED(n_token_count_out); + + +// TODO: implement with GGUF format + return true; +} + +bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { + try { + return llama_load_session_file_internal(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out); + } catch (const std::exception & err) { + fprintf(stderr, "error loading session file: %s\n", err.what()); + return false; + } +} + +bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { + gguf_file file(path_session, "wb"); + + // TODO: implement with GGUF format + + return true; +} + +int llama_eval( + struct llama_context * ctx, + const llama_token * tokens, + int n_tokens, + int n_past, + int n_threads) { + if (!llama_eval_internal(*ctx, tokens, nullptr, n_tokens, n_past, n_threads, nullptr)) { + fprintf(stderr, "%s: failed to eval\n", __func__); + return 1; + } + + // get a more accurate load time, upon first eval + // TODO: fix this + if (!ctx->has_evaluated_once) { + ctx->t_load_us = ggml_time_us() - ctx->t_start_us; + ctx->has_evaluated_once = true; + } + + return 0; +} + + +int llama_eval_embd( + struct llama_context * ctx, + const float * embd, + int n_tokens, + int n_past, + int n_threads) { + if (!llama_eval_internal(*ctx, nullptr, embd, n_tokens, n_past, n_threads, nullptr)) { + fprintf(stderr, "%s: failed to eval\n", __func__); + return 1; + } + + // get a more accurate load time, upon first eval + // TODO: fix this + if (!ctx->has_evaluated_once) { + ctx->t_load_us = ggml_time_us() - ctx->t_start_us; + ctx->has_evaluated_once = true; + } + + return 0; +} + +int llama_eval_export(struct llama_context * ctx, const char * fname) { + const int n_batch = 1; + const int n_ctx = 512 - n_batch; + + const std::vector tmp(n_batch, llama_token_bos()); + + if (!llama_eval_internal(*ctx, tmp.data(), nullptr, tmp.size(), n_ctx, 1, fname)) { + fprintf(stderr, "%s: failed to eval\n", __func__); + return 1; + } + + return 0; +} + +int llama_tokenize_with_model( + const struct llama_model * model, + const char * text, + llama_token * tokens, + int n_max_tokens, + bool add_bos) { + auto res = llama_tokenize(model->vocab, text, add_bos); + + if (n_max_tokens < (int) res.size()) { + fprintf(stderr, "%s: too many tokens\n", __func__); + return -((int) res.size()); + } + + for (size_t i = 0; i < res.size(); i++) { + tokens[i] = res[i]; + } + + return res.size(); +} + +int llama_tokenize( + struct llama_context * ctx, + const char * text, + llama_token * tokens, + int n_max_tokens, + bool add_bos) { + return llama_tokenize_with_model(&ctx->model, text, tokens, n_max_tokens, add_bos); +} + +int llama_n_vocab_from_model(const struct llama_model * model) { + return model->vocab.id_to_token.size(); +} + +int llama_n_ctx_from_model(const struct llama_model * model) { + return model->hparams.n_ctx; +} + +int llama_n_embd_from_model(const struct llama_model * model) { + return model->hparams.n_embd; +} + +int llama_n_vocab(const struct llama_context * ctx) { + return ctx->model.vocab.id_to_token.size(); +} + +int llama_n_ctx(const struct llama_context * ctx) { + return ctx->model.hparams.n_ctx; +} + +int llama_n_embd(const struct llama_context * ctx) { + return ctx->model.hparams.n_embd; +} + +int llama_get_vocab_from_model( + const struct llama_model * model, + const char * * strings, + float * scores, + int capacity) { + int n = std::min(capacity, (int) model->vocab.id_to_token.size()); + for (int i = 0; ivocab.id_to_token[i].tok.c_str(); + scores[i] = model->vocab.id_to_token[i].score; + } + return n; +} + +int llama_get_vocab( + const struct llama_context * ctx, + const char * * strings, + float * scores, + int capacity) { + return llama_get_vocab_from_model(&ctx->model, strings, scores, capacity); +} + +float * llama_get_logits(struct llama_context * ctx) { + return ctx->logits.data(); +} + +float * llama_get_embeddings(struct llama_context * ctx) { + return ctx->embedding.data(); +} + +const char * llama_token_to_str_with_model(const struct llama_model * model, llama_token token) { + if (token >= llama_n_vocab_from_model(model)) { + return nullptr; + } + + return model->vocab.id_to_token[token].tok.c_str(); +} + +const char * llama_token_to_str(const struct llama_context * ctx, llama_token token) { + return llama_token_to_str_with_model(&ctx->model, token); +} + +llama_token llama_token_bos() { + return 1; +} + +llama_token llama_token_eos() { + return 2; +} + +llama_token llama_token_nl() { + return 13; +} + +struct llama_timings llama_get_timings(struct llama_context * ctx) { + struct llama_timings result = { + /*.t_start_ms =*/ 1e-3 * ctx->t_start_us, + /*.t_end_ms =*/ 1.00 * ggml_time_ms(), + /*.t_load_ms =*/ 1e-3 * ctx->t_load_us, + /*.t_sample_ms =*/ 1e-3 * ctx->t_sample_us, + /*.t_p_eval_ms =*/ 1e-3 * ctx->t_p_eval_us, + /*.t_eval_ms =*/ 1e-3 * ctx->t_eval_us, + + /*.n_sample =*/ std::max(1, ctx->n_sample), + /*.n_p_eval =*/ std::max(1, ctx->n_p_eval), + /*.n_eval =*/ std::max(1, ctx->n_eval), + }; + + return result; +} + +void llama_print_timings(struct llama_context * ctx) { + const llama_timings timings = llama_get_timings(ctx); + + fprintf(stderr, "\n"); + fprintf(stderr, "%s: load time = %8.2f ms\n", __func__, timings.t_load_ms); + fprintf(stderr, "%s: sample time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", + __func__, timings.t_sample_ms, timings.n_sample, timings.t_sample_ms / timings.n_sample, 1e3 / timings.t_sample_ms * timings.n_sample); + fprintf(stderr, "%s: prompt eval time = %8.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", + __func__, timings.t_p_eval_ms, timings.n_p_eval, timings.t_p_eval_ms / timings.n_p_eval, 1e3 / timings.t_p_eval_ms * timings.n_p_eval); + fprintf(stderr, "%s: eval time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", + __func__, timings.t_eval_ms, timings.n_eval, timings.t_eval_ms / timings.n_eval, 1e3 / timings.t_eval_ms * timings.n_eval); + fprintf(stderr, "%s: total time = %8.2f ms\n", __func__, (timings.t_end_ms - timings.t_start_ms)); +} + +void llama_reset_timings(struct llama_context * ctx) { + ctx->t_start_us = ggml_time_us(); + ctx->t_sample_us = ctx->n_sample = 0; + ctx->t_eval_us = ctx->n_eval = 0; + ctx->t_p_eval_us = ctx->n_p_eval = 0; +} + +const char * llama_print_system_info(void) { + static std::string s; + + s = ""; + s += "AVX = " + std::to_string(ggml_cpu_has_avx()) + " | "; + s += "AVX2 = " + std::to_string(ggml_cpu_has_avx2()) + " | "; + s += "AVX512 = " + std::to_string(ggml_cpu_has_avx512()) + " | "; + s += "AVX512_VBMI = " + std::to_string(ggml_cpu_has_avx512_vbmi()) + " | "; + s += "AVX512_VNNI = " + std::to_string(ggml_cpu_has_avx512_vnni()) + " | "; + s += "FMA = " + std::to_string(ggml_cpu_has_fma()) + " | "; + s += "NEON = " + std::to_string(ggml_cpu_has_neon()) + " | "; + s += "ARM_FMA = " + std::to_string(ggml_cpu_has_arm_fma()) + " | "; + s += "F16C = " + std::to_string(ggml_cpu_has_f16c()) + " | "; + s += "FP16_VA = " + std::to_string(ggml_cpu_has_fp16_va()) + " | "; + s += "WASM_SIMD = " + std::to_string(ggml_cpu_has_wasm_simd()) + " | "; + s += "BLAS = " + std::to_string(ggml_cpu_has_blas()) + " | "; + s += "SSE3 = " + std::to_string(ggml_cpu_has_sse3()) + " | "; + s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | "; + + return s.c_str(); +} + +// For internal test use +const std::vector>& llama_internal_get_tensor_map(struct llama_context * ctx) { + return ctx->model.tensors_by_name; +} diff --git a/gguf-llama.h b/gguf-llama.h new file mode 100644 index 0000000000000..20dcc9f639510 --- /dev/null +++ b/gguf-llama.h @@ -0,0 +1,468 @@ +#ifndef LLAMA_H +#define LLAMA_H + +#include "ggml.h" +#ifdef GGML_USE_CUBLAS +#include "ggml-cuda.h" +#define LLAMA_MAX_DEVICES GGML_CUDA_MAX_DEVICES +#else +#define LLAMA_MAX_DEVICES 1 +#endif // GGML_USE_CUBLAS +#include +#include +#include + +#ifdef LLAMA_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef LLAMA_BUILD +# define LLAMA_API __declspec(dllexport) +# else +# define LLAMA_API __declspec(dllimport) +# endif +# else +# define LLAMA_API __attribute__ ((visibility ("default"))) +# endif +#else +# define LLAMA_API +#endif + +#ifdef __GNUC__ +# define DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define DEPRECATED(func, hint) func +#endif + +#define LLAMA_FILE_MAGIC_GGJT 0x67676a74u // 'ggjt' +#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' +#define LLAMA_FILE_MAGIC_GGMF 0x67676d66u // 'ggmf' +#define LLAMA_FILE_MAGIC_GGML 0x67676d6cu // 'ggml' +#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' + +#define LLAMA_FILE_VERSION 3 +#define LLAMA_FILE_MAGIC LLAMA_FILE_MAGIC_GGJT +#define LLAMA_FILE_MAGIC_UNVERSIONED LLAMA_FILE_MAGIC_GGML +#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN +#define LLAMA_SESSION_VERSION 1 + +#define LLAMA_DEFAULT_SEED 0xFFFFFFFF + +#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL) +// Defined when llama.cpp is compiled with support for offloading model layers to GPU. +#define LLAMA_SUPPORTS_GPU_OFFLOAD +#endif + +#ifndef LLAMA_DEFAULT_RMS_EPS +#define LLAMA_DEFAULT_RMS_EPS 5e-6f +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + // + // C interface + // + // TODO: show sample usage + // + + struct llama_model; + struct llama_context; + + typedef int llama_token; + + typedef struct llama_token_data { + llama_token id; // token id + float logit; // log-odds of the token + float p; // probability of the token + } llama_token_data; + + typedef struct llama_token_data_array { + llama_token_data * data; + size_t size; + bool sorted; + } llama_token_data_array; + + typedef void (*llama_progress_callback)(float progress, void *ctx); + + struct llama_context_params { + uint32_t seed; // RNG seed, -1 for random + int32_t n_ctx; // text context + int32_t n_batch; // prompt processing batch size + int32_t n_gqa; // grouped-query attention (TEMP - will be moved to model hparams) + float rms_norm_eps; // rms norm epsilon (TEMP - will be moved to model hparams) + int32_t n_gpu_layers; // number of layers to store in VRAM + int32_t main_gpu; // the GPU that is used for scratch and small tensors + + const float * tensor_split; // how to split layers across multiple GPUs (size: LLAMA_MAX_DEVICES) + + // ref: https://github.com/ggerganov/llama.cpp/pull/2054 + float rope_freq_base; // RoPE base frequency + float rope_freq_scale; // RoPE frequency scaling factor + + // called with a progress value between 0 and 1, pass NULL to disable + llama_progress_callback progress_callback; + // context pointer passed to the progress callback + void * progress_callback_user_data; + + // Keep the booleans together to avoid misalignment during copy-by-value. + bool low_vram; // if true, reduce VRAM usage at the cost of performance + bool f16_kv; // use fp16 for KV cache + bool logits_all; // the llama_eval() call computes all logits, not just the last one + bool vocab_only; // only load the vocabulary, no weights + bool use_mmap; // use mmap if possible + bool use_mlock; // force system to keep model in RAM + bool embedding; // embedding mode only + }; + // model file types + enum llama_ftype { + LLAMA_FTYPE_ALL_F32 = 0, + LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed + // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed + LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K = 10,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_S = 11,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_M = 12,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_L = 13,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_S = 14,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_M = 15,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_S = 16,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_M = 17,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q6_K = 18,// except 1d tensors + }; + + // model quantization parameters + typedef struct llama_model_quantize_params { + int nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + } llama_model_quantize_params; + + // grammar types + struct llama_grammar; + + // grammar element type + enum llama_gretype { + // end of rule definition + LLAMA_GRETYPE_END = 0, + + // start of alternate definition for rule + LLAMA_GRETYPE_ALT = 1, + + // non-terminal element: reference to rule + LLAMA_GRETYPE_RULE_REF = 2, + + // terminal element: character (code point) + LLAMA_GRETYPE_CHAR = 3, + + // inverse char(s) ([^a], [^a-b] [^abc]) + LLAMA_GRETYPE_CHAR_NOT = 4, + + // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to + // be an inclusive range ([a-z]) + LLAMA_GRETYPE_CHAR_RNG_UPPER = 5, + + // modifies a preceding LLAMA_GRETYPE_CHAR or + // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) + LLAMA_GRETYPE_CHAR_ALT = 6, + }; + + typedef struct llama_grammar_element { + enum llama_gretype type; + uint32_t value; // Unicode code point or rule ID + } llama_grammar_element; + + // performance timing information + struct llama_timings { + double t_start_ms; + double t_end_ms; + double t_load_ms; + double t_sample_ms; + double t_p_eval_ms; + double t_eval_ms; + + int32_t n_sample; + int32_t n_p_eval; + int32_t n_eval; + }; + + LLAMA_API int llama_max_devices(); + + LLAMA_API struct llama_context_params llama_context_default_params(); + LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(); + + LLAMA_API bool llama_mmap_supported(); + LLAMA_API bool llama_mlock_supported(); + + // TODO: not great API - very likely to change + // Initialize the llama + ggml backend + // If numa is true, use NUMA optimizations + // Call once at the start of the program + LLAMA_API void llama_backend_init(bool numa); + // Call once at the end of the program - currently only used for MPI + LLAMA_API void llama_backend_free(); + + LLAMA_API int64_t llama_time_us(); + + LLAMA_API struct llama_model * llama_load_model_from_file( + const char * path_model, + struct llama_context_params params); + + LLAMA_API void llama_free_model(struct llama_model * model); + + LLAMA_API struct llama_context * llama_new_context_with_model( + struct llama_model * model, + struct llama_context_params params); + + // Various functions for loading a ggml llama model. + // Allocate (almost) all memory needed for the model. + // Return NULL on failure + LLAMA_API DEPRECATED(struct llama_context * llama_init_from_file( + const char * path_model, + struct llama_context_params params), + "please use llama_load_model_from_file combined with llama_new_context_with_model instead"); + + // Frees all allocated memory + LLAMA_API void llama_free(struct llama_context * ctx); + + // Returns 0 on success + LLAMA_API int llama_model_quantize( + const char * fname_inp, + const char * fname_out, + const llama_model_quantize_params * params); + + // Apply a LoRA adapter to a loaded model + // path_base_model is the path to a higher quality model to use as a base for + // the layers modified by the adapter. Can be NULL to use the current loaded model. + // The model needs to be reloaded before applying a new adapter, otherwise the adapter + // will be applied on top of the previous one + // Returns 0 on success + LLAMA_API DEPRECATED(int llama_apply_lora_from_file( + struct llama_context * ctx, + const char * path_lora, + const char * path_base_model, + int n_threads), + "please use llama_model_apply_lora_from_file instead"); + + LLAMA_API int llama_model_apply_lora_from_file( + const struct llama_model * model, + const char * path_lora, + const char * path_base_model, + int n_threads); + + // Returns the number of tokens in the KV cache + LLAMA_API int llama_get_kv_cache_token_count(const struct llama_context * ctx); + + // Sets the current rng seed. + LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed); + + // Returns the maximum size in bytes of the state (rng, logits, embedding + // and kv_cache) - will often be smaller after compacting tokens + LLAMA_API size_t llama_get_state_size(const struct llama_context * ctx); + + // Copies the state to the specified destination address. + // Destination needs to have allocated enough memory. + // Returns the number of bytes copied + LLAMA_API size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst); + + // Set the state reading from the specified address + // Returns the number of bytes read + LLAMA_API size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src); + + // Save/load session file + LLAMA_API bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out); + LLAMA_API bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count); + + // Run the llama inference to obtain the logits and probabilities for the next token. + // tokens + n_tokens is the provided batch of new tokens to process + // n_past is the number of tokens to use from previous eval calls + // Returns 0 on success + LLAMA_API int llama_eval( + struct llama_context * ctx, + const llama_token * tokens, + int n_tokens, + int n_past, + int n_threads); + + // Same as llama_eval, but use float matrix input directly. + LLAMA_API int llama_eval_embd( + struct llama_context * ctx, + const float * embd, + int n_tokens, + int n_past, + int n_threads); + + // Export a static computation graph for context of 511 and batch size of 1 + // NOTE: since this functionality is mostly for debugging and demonstration purposes, we hardcode these + // parameters here to keep things simple + // IMPORTANT: do not use for anything else other than debugging and testing! + LLAMA_API int llama_eval_export(struct llama_context * ctx, const char * fname); + + // Convert the provided text into tokens. + // The tokens pointer must be large enough to hold the resulting tokens. + // Returns the number of tokens on success, no more than n_max_tokens + // Returns a negative number on failure - the number of tokens that would have been returned + // TODO: not sure if correct + LLAMA_API int llama_tokenize( + struct llama_context * ctx, + const char * text, + llama_token * tokens, + int n_max_tokens, + bool add_bos); + + LLAMA_API int llama_tokenize_with_model( + const struct llama_model * model, + const char * text, + llama_token * tokens, + int n_max_tokens, + bool add_bos); + + LLAMA_API int llama_n_vocab(const struct llama_context * ctx); + LLAMA_API int llama_n_ctx (const struct llama_context * ctx); + LLAMA_API int llama_n_embd (const struct llama_context * ctx); + + LLAMA_API int llama_n_vocab_from_model(const struct llama_model * model); + LLAMA_API int llama_n_ctx_from_model (const struct llama_model * model); + LLAMA_API int llama_n_embd_from_model (const struct llama_model * model); + + // Get the vocabulary as output parameters. + // Returns number of results. + LLAMA_API int llama_get_vocab( + const struct llama_context * ctx, + const char * * strings, + float * scores, + int capacity); + + LLAMA_API int llama_get_vocab_from_model( + const struct llama_model * model, + const char * * strings, + float * scores, + int capacity); + + // Token logits obtained from the last call to llama_eval() + // The logits for the last token are stored in the last row + // Can be mutated in order to change the probabilities of the next token + // Rows: n_tokens + // Cols: n_vocab + LLAMA_API float * llama_get_logits(struct llama_context * ctx); + + // Get the embeddings for the input + // shape: [n_embd] (1-dimensional) + LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); + + // Token Id -> String. Uses the vocabulary in the provided context + LLAMA_API const char * llama_token_to_str( + const struct llama_context * ctx, + llama_token token); + + LLAMA_API const char * llama_token_to_str_with_model( + const struct llama_model * model, + llama_token token); + + // Special tokens + LLAMA_API llama_token llama_token_bos(); // beginning-of-sentence + LLAMA_API llama_token llama_token_eos(); // end-of-sentence + LLAMA_API llama_token llama_token_nl(); // next-line + + // Grammar + // + LLAMA_API struct llama_grammar * llama_grammar_init( + const llama_grammar_element ** rules, + size_t n_rules, + size_t start_rule_index); + + LLAMA_API void llama_grammar_free(struct llama_grammar * grammar); + + // Sampling functions + + /// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. + LLAMA_API void llama_sample_repetition_penalty(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float penalty); + + /// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. + LLAMA_API void llama_sample_frequency_and_presence_penalties(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float alpha_frequency, float alpha_presence); + + /// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806 + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, the logits must be directly extracted from the original generation context without being sorted. + /// @params guidance_ctx A separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context. + /// @params scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance. + LLAMA_API void llama_sample_classifier_free_guidance( + struct llama_context * ctx, + llama_token_data_array * candidates, + struct llama_context * guidance_ctx, + float scale); + + /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. + LLAMA_API void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates); + + /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + LLAMA_API void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep); + + /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + LLAMA_API void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep); + + /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. + LLAMA_API void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep); + + /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. + LLAMA_API void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep); + LLAMA_API void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates, float temp); + + /// @details Apply constraints from grammar + LLAMA_API void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * candidates, const struct llama_grammar * grammar); + + /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int m, float * mu); + + /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu); + + /// @details Selects the token with the highest probability. + LLAMA_API llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates); + + /// @details Randomly selects a token from the candidates based on their probabilities. + LLAMA_API llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates); + + /// @details Accepts the sampled token into the grammar + LLAMA_API void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar * grammar, llama_token token); + + // Performance information + LLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx); + LLAMA_API void llama_print_timings(struct llama_context * ctx); + LLAMA_API void llama_reset_timings(struct llama_context * ctx); + + // Print system information + LLAMA_API const char * llama_print_system_info(void); + +#ifdef __cplusplus +} +#endif + +// Internal API to be implemented by llama.cpp and used by tests/benchmarks only +#ifdef LLAMA_API_INTERNAL + +#include +#include +struct ggml_tensor; + +const std::vector>& llama_internal_get_tensor_map(struct llama_context * ctx); + +#endif + +#endif // LLAMA_H From 4f865181aa9b3f214ed772688de5ad24068a5f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Thu, 10 Aug 2023 17:49:31 +0300 Subject: [PATCH 086/242] gguf : start implementing libllama in GGUF (WIP) --- gguf-llama-simple | Bin 607488 -> 607488 bytes gguf-llama.cpp | 5 ++++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/gguf-llama-simple b/gguf-llama-simple index d7600282e3315a4bc805a4f6c01550d58bdb80a5..bfa61de57dbc77cfb6c180b20a23a46dfebaf99f 100644 GIT binary patch delta 19466 zcmb_^34Bb~`}f?N%$XZRV#y{Ui-{~m7D9rM5KI{R7Hd%rT5UuVJCllSI%4P0s--_g zRaV-v9e~KkqxA&*Zu1`#k44&-0x9oSVu0 zI5hX;(Cq@}_Qg%2W1lZ}*5U%gxiiaRBYZ5|?Fx7%Ly(!uU!5-KGX+2Y2EMqCL6#HW zz=t1GZR!P9Q{ul)-QVm3w0M24MA^kJx?ll_#1e# zpQ0_{4Ls<&BA@&QenwPq(;IlQQm#eU^9_b)fKtHlH}K?FilQ-Z;JJE5e*7D_XQ?7T z`3*cKP?29Op36=DwqiFC!TgK|i?6CGoNM=6!vc!8IA^&fA-}*B=k#bwAi0WA*1a?y>#%_w@c8H;Xmg>LdGVS-4>J2Wu@8|YsEA(@CK5H?Zu}Z&(n@yXq z)`xS8=+Ma?zU(e|0)#tvm55BMO?!xzp zr(ZqNujh79^J9Gy|6y~x?XkWO&pBwx-}*V+eEU03^+g}9e(NwZ2Oo5fQsxbQ-!+hyE~C+S%0Y@^v@_HaUbdy|G~6%PuEDlN&rrsCvqCTJ?2ZCuYD z@JT~qCaaQ99EL*EY=_nZH=*!wll7Q*$5JVotVK3ms%-{6=uLvVmYeoT$|ukxZ!u)M zSzo0^6q#(Zjq{N*M_e@>F6YgOqDS5=q|>aI(+=O|&DN`?*y9YHiNE9&m=nvh z{a;gPGZ%TLSZmBRNd;+%#W@dC@;krED|dLzl3a8;TdHwzSA>wI*Wh@yJB1VjW6*@4mVq$i#rlfJNeU%Jz1N$ zXtq8|vlfVVlPV?K5aW1jiTF#uN-51+Y_ge+lbM2C7J<~phTu;Kf<>nM zY@-l+R!nWn3MQ;zAl#gtg=kypU$bmDxbSuScuJ>qBaO*<^{Y1U!k z_{Ma}cl{m(qDr!jlg(-|X%Ww}tDCGPW^1Lms!gS2=3X#aQ+}#(IND8eOyS{~mQKkm zXlZ%JhIg|fc#U(=p~e{n6Ji!_Vl6fVWwTnI*n)MhqpZ6no2;V6l=lL$LdTvE#OhUG z@k~ZMPopFz1Qli`X*T+r=3L5d40$fci-F-Dd)yFGzYJ|SVD+BOdhtc^YD*=4$eC*N zlxu~BkcEzoC~~cdtqnShF72>GZ`$|OR^?;%`1Vnn^t0I-!%P=Ui4U^%qLDQ^BqzfY zB)!yRPdNlEva=^tN|N{rWtNV?lNsa7c9haF_F32bHk+LN*bUlK5a4Fq!rzmTLu zp1@gb!{S*RJZ8#|4;Tku<#5{i8pBhq6|n`Pb+;rX@8)FJSy%cJ154~ZlPx~E50Aij zUF*$b$ayYa173O>D(O9c;%g~r?o>*gO3@O`)@x8{d_!fbQ{^S-g39Tt%IUyO`AL&U zbF!U3z)n$sUSM5-iQDk3dC@~D5QdM3CzkB@R98^m?p^DZ|C@x4Psb7_NOQQXqDVS^h1@DwJEDr7R8v&*q^5OV=0T! z7UXsr7t~EpDg0)(t!;~nKz3WJV{G@=`q`k zp3N%#?`CvRAUPQu-i-0ur6oG5cy)o5wu7MAW;80kqI`j6tB9lhf=G+bF+qs`Pcg8^ zS}dl5cAN7or>4o)#%QwM7vJV|x-vAx=i=%()LC6-xAmwPAa~*tF*+YY!ESV zy{UgNX~N}DQ!weypNpldf-&3k^v7Tl)^2}Or6Ji`o8?~BC6+bq=VBa`W-La4o|tSi z)c`dG@1Fq*ps^t&hW{#xz8ym5^8bmUWg%obr>A2Y5-We{C;FrzN#wW3(56PRJF=p& ztlMP0B7OiD#0#>6N5yMQFX|={-Y|y`|47|KNkXkK*|uUpvCqubJ7P=PKa?CA(=v*w z`49nUJve1Z%FvWyDQ_JWKZ;Z`e+imqb48#!go_KJ8F*A&E|*jy-c>`IeNvJ<0+V&B ze;D67scqVXXb&&5o7r4b3_7e!L35ecXp zt7J3f6$;EISXA18Mu!ovKmnX?)>~}sSli9kD9tugNK;F@DwZ{-Z-$hxDK zTQ^Q;&etc5;B@=;V`wi!^U|N!q|PZ)f0<<|9h2GvDRf~oLW=Nagp=v~$C30#IBCqy zpq>#VxZ&V%1jJU?Xqt0&W_5XCM3CqF+;A#Jl7xV5Xbau*M?KZ0yh8wjycZNH0NHnV6r|CLulis zhDDPGKJ&s5!~B4C=+~}V^*wG#heVSMbA>02UJ=ZUy`YXLSWTVAXfVE^?17k$ z?9RDN9WR2E@uEAEuuT(K3WG&ES~G^s;IB2J`7tC4{pQbK@S zVzIb!H;rsfR`Xx_(4yAlFn`RO&Tm7S7z|iLi2tcezi&gjaigeP5_z7sr!BAnb(kr~ zFaV>Dal0*-xUO~xh}{zXSbz?{a?w@@^Xt%K?TFcJwN9B_+f)B!a)e)0o0cS#ru-o<>XAac zxNsVpLVn_=(Z?wyr2gKzl~U0QPb{yx=a0LGo>3&Q2Fq30H0G38(vW zh%p=mVYQYfozB@>^kUfIqKc;aXAZi)0~u9qF|-st8gK%GT+LJx8{m#Ch1X{cbg!X4 z)c1d-laWx~J~);9!3ETdWFk`ry8EdQUg0m8$drNgJ?z~(k**x~A^oH?Ce62Lq>1eJ z9aD{k9&J(_hNsxGWhYrPIJ=p?G!b7eoqDB_7F;kjr;!*gna)Tfv2jfeN^@B0hN`Po zgC&lNe(nmGCDUTDVv|%gzppfsCj6Od%IxDuwAtt26)X%Zm<7H@Grw#o`H2ts!rNrL2fua#1B>jp~Zf@{RKJL!;v{ zg?!_p^x`jx(u+sajBXfyr_rU|$UH=?UUxE$>q=L4$K)+;ptrh{ATEO%dXP3;1WoOM z_?T!;4-y;Mf@9iC#I-Jn5A&849jpdzicE_|Zi6H(rsKPjVBTLxmvzIeUB`v`cPCT% zvJm>bC+Wt|UndvB6P^k^NNfIto_6d(qIpqAC!)B>bB+qRoVF-W+=s$gIfrHcOGH1t z;^Kz2teoT!672H*s|q?Qgv{Z0>XhYy7ftVj-V02c_dRBRg!ySdTS=o2U{PGV9%j6J_pi&b(YBt)#5E9W&y>ABY-YATg-q(>8eg;bh_?HnE77e@&DU(gkONldMT z3Z@o&U+gAw>WL>_(RT)rR{XsRx?uqE<(Ivn-wz-W-NwMxZdM1INwG9&`_H98S1`*i zU@h*`4EA?}Mlqe0_A7iX^FqkkP9j~b+e}&4)}XQJ#M?#xLN?yZ+a#7%GV__tFL8>@ z2?xzdCn<)xs6cT^CH(=3pzgBR0eQnC20wS^2~+a~(gp)DnGB_=1Ic!rLp>cxI`KR7 zG-(jITl?ru3=|cX*+x(7-?7yAwGM4Pm<;qye_DYqBFU?^6ziYZmaqeXrImEsVC-Ga zmsdzQP?{a&#;O%I#{0%8qvQvFE1Zcqu2eW2IN|^meoKQN#R5)NVP8$ppA3#rVNQcL zGbOcE*kq~ZRKAaizj&gQ8LPoxG5CfGU%}z2VhUyODHT4e!HXEYM}>_woYFQ1uT$aO z8ay|avtYNc;sy<$z{Imu_@SoUn^`_cg+J1y$1&*+D*Ubnk6>^U74~#@n$nxWUMjq< zhQ*1uWw_(9;{JS1OEU)FR^bRui$8Src4Ia(lI2FF7!9y7Aufp8{TNEuOhP$eGOHI5jgYP|5-1oq) zS&?qa;BzXRslh=EKBU5pHMjDCuw*o!>6k7a1Fl5;I~zHq6VL2 zuvvw_)Zjx5j#uH2H25n9H&Ees8Elbne+@PC3^ zG&qjI^nQg!9{d+*_)Lc9s^Xt$@B{{rRpC4hKEUAKD%?kFw%;)6I7gG}O!@6vf0q;XWMJ8Zo+ z-DrAa#WDdMq;$jI5>~jy*ygb_sFE^Tl0l3OPKf`5`XMxUH0jy;Qdxy$>)<-4z+`J; z^jP%p33eTu5p`<^*V4a`Fj5+Tg=iG zneMAXsrMN2Uj6q_C_5}KXQwvoNUUmcwf-JmJBAGAciyKj#*mTKN|;O73>4p@qsNjY zKKwR4{4VjL-;X5?u&KB>7Dp6CwBTLhO&h#RI&kkP&y(LJojG^*Je2?TCJlU#4CMOK zDesX%e8mm=`+MYFKK}+y8Ap=1?$k04J7bRi1j6t7i`IJ|+dS-E-X}dcAG+s#peL_W z*AIZspgli;Yz+PE1JZ-PdX3%($}hV{TW6Ba{3kc*>`W5P`O#gOBpy4&JDH>lzv?P& zF`hK%uV1I*#$z)WN&hpRbmW@T((yQQ+*v~XvygZHUvyL!Y0T&UMVDmZu(Jcbm_;@< zTmF(=D^x$&AGN(7>v!xLl1|k}Q|oh+g|Pa;wAUEb8-US@?wmlP{o~|}!ap+lGybC+ z^wk7X-%|O4X%-F+dQCw$Ca~`Y8cTwRYGpS>$8sB^lDN`fY!0w8E>v?3eJxiW%azA+ zy?iMPty6`Dye9M`6B@>Z{*Z;}>JLdv{^ezQ_Cqw_yvwv}HqH-y=)`PdZc^&OXBmS(? zByPM+D|5&P{H`nX-HC|xyepJWB;m0`ufVKrH*nX1g*nEbi>#Oh#_e$2WW`AwE`H*B zm9ubpy$d4Nrw=C*U&C(~u>Ck)LOmvtPq{(#t4U-ZU;YQpm`nnz{Z)rU%PtG>2&9K5 zkuPvS5G?xo zJ7@Z+FkP?HX~)Jxao3aUAE!u~aB%{2;%lP>NZ!9l3#Jm^jJ~qjNilAS z&g4H>?xYrMLH!eRG(RblJr}UB<|ox)GJO>pUxsf{arV4M4vhZZ6Xppn3YEep`pSi6 z(zeq`0KfkU9Wo77{LOiqJB=7Uw#d4a#v+!Tv+taS(^mfGS(-KjgKfd@bm0sVXj#YH zV4GkT*f_cK{4=mKA;1nUGvEp?R#LGu@81WvIKP^0@z)=6xY}=PvU$Vxw%)Qe-=J8W zXCIQs;S|`7dsMg=5AH$i*woM;hc?Er*^oCJ;Yt;)$y>cki^8e^r#7{(agHO35$ zv79lqo-dBl7)6ZnmX@Qd#<0}z0Vh=B#AzHq#?Xcr(NALxV2qwx4xPqW#2CXg#_jV? z-+yNelg9X2X0)~SaW&;z;(4VKr)rT(1#qywCcQyZlg1cT-p$t-QyHVTmLpSR>|%^6 zANp&I$BZ!$IV?&~NYXgXSQn|vV$>K}jL}ysq`JoVnlY;C@3CC}tjbUQQ`H5HQN12= zbd^*oHwJiM^W3Z;HyT(|}7$Y=ByvDfB7;zdSKx4dQj3ABSqA@J4xVwFA^tn?N z>f;bsYqBYX5QF@)a*JZEr@6{`OW}N2#Ys>(vlPxinZqKmhm|x&vq5Py7RDjL;Iz;< zqh%Wy!^jxv8l$_$7{wTw8Y8~S>iGaw3%-RAtUmvXvw{4|a#fAA5^Bw&jOT74;8aEQ zmQvX!+cHnKI>?Xj3O-uMLRl619@ZxDs}*dj6AJ)#Tfs7bi79;@((Hn4Kc$2Q4N*c> zEzd>?qDNnc8Y4zybim!|!5$jJM`KK7jChUV zuxQmo8Ka8n zajM4ogmLO=Sw_eV%*e~m07HCNfRB1`JHx6}nU$4G>|v!Vey028kYuhUt(-%;)(S%v zn5-`HTRTPNp8A^h{&TTA;D7j$?wCis_+uyN@p-tE@TL0sBsn4LZ`e`$1k_rc*J^t4 z@kzE=crIT28H)w&)i<_aNW<$_))(TA9(2Zh@&o_4J8fenm-*k;(0U)^6@t%4!qwnfHt*}1eZp~D(IBw`jGI$#T63XA$u{!OGqo1 zE=3izQxDRV8%z5xB@xZuI;e;j7oZQfSHdH9QH&BlmuayWISe;@TAQJsp(MWcJY;zN zu9+UB&zF*e9jR>Bo3DsmJHU!57Ttbeja_|3p(xT>x&1mWG?5#kEQkD3)Z2!K3K1d+@ z!=CI>v<}_}&(3|X6m>XZn7njM;KoCE{h77K@|$DOZk*T9SD#6QS5Ifi!d(zq^9A!$Wu%=)mRmzupp< z0Cn1%9yj+?(2o;v;r`Zbczk#l{cH{C)UuZc4k=#Nnr&Q1#~V>k zJ?W{9q<(F0tW9$GX$L**t&7SdT#aI|{pCh-i{rO$r8hR=j6zRiHj_s5 z+I0)*&GoRa-9lP&oCiJkB~DI~>C-RCGu&0)-AelKU$3yIe1)toTO=J_w2i!U`O=}I z_qW0(cl*_^+3AQtqrN2$7q^!x{=PkTJHyA&Q#;6WZagjej`Zb!&A0pS#H7t{UqIXL zBE9MiT>w|hBT|h)u?2-AXQX>=d`>^#MPh@~wpU0+m9kpFFq!y^9>>Jv&-e1;cR(pRR8~i|r&a>A+JR;(RYWbhfc4f_Z1;qnz-pUB<@B$b2dr9#{nat{MA6j& zmtc%O!+!bzY0dH1XVIZYh(8TGNS^R#KT;Ac4mxc|F8pgO|7(q+(o0^ofRB8V zBZPW++xHwJ^<6A;uET=zh}rU`Dzi!7gCVSwJj4(xBkwqo7-yFl;6&IG3AN^{_nM^Y zS?|nM-6^x!nJd(ZjCUdrxNEe}Nu2Kuv43=utm66aCQ$!VaQ50&nsf?P_}3L$d5T0v zG`|A%N0gwxTv;KdSc@{+TV0zKReGlyd!kD9wILy1TS>e9Og6`cGZBwHg+==#j~7)A zJX}=XR9~KjUPU}?L-0By+M{v2>~=296)RTQLr;@%j$b#P_Bcc0>U_HV^`?vW2|iwK zUw(#Ux>@?2eKTHx4EaaA;+;s0)6Xy`qQ&cG#+&iF^oy!Ti&upc(c*Q8`-SFRBq`Cp zSSpTuG9$e0k7VQ{%a@r8Ts+}O(bF>bVYqb%6k$;z3{7Qc>642jjB8I9T_SHL zh0xQNh#$?qMCx%h8QzY+f0mxPMB@01XX)ZgqyY`NjEdNQmZn|?xaq8Y@?{*(^4=qq z8tQmjsiDrNS7rtmlrar3({27_wcG~1A$>F-%KYxmoZ_eL-_8Hbg++7-5 zL2~HCatxb}FG*J5jBgR8>odaB zy(T%I>SsBhLf@keUXkJa+R-%k6=|N34hGsmQj9|5)pS&{k`5Jm;D2uNX9pIj)qWgN ze3j`HYrp%7OmpFPr`X4GhI=kCn_4r&J>9)G%hvgNo!X>4)IW{~7I@Yuqh7E?rDyzG z_PKh)F?W7_6zyKe5XEokMQ7A81oFAP=-N64AM923)G;*0diZ7?L%U|_pF&srj=H0) zhTl^YKNbD(^wP-3@!`EM1CGjO@R5$Xn@GgZ`_w+5t|67 zmHApTM|!LR6vx5|r7Wu0>^D$07|0_L;Ak^OY~h-E7GS$W}+p{^OO zg-5!%PJ}7_bV*V4VSu4uo##*>%hrS}d+-XS>GcgCR{w=DAQ&YswV$hR@Z)MdU4nAi zput87KmFW>D+3LY9UphBkkAn_dps<9H2fC?Psj{yx)9}l_r9RJET^Ni^i4S(uchzH z>7=GKGtjV$-?z{n8f5U~xz6?$AqaMdZC%(7vo&^v?ep1A*~MuZvy@OEyeP{SDtw~?WvYm`r_yt75QULmr6GEdfj1;;G=5Pj%poqT>IiulkT+1PN0=f>E* zni|HsG+vqH%)8uQ&O0Yp&iggUzVnK0)r`paG?0E7Yl!CR+AqW!2Jw;CS~)Y8_{tg2 zV7&#WwqRIM3l;kXBV#fh+T37n^cQYzQ~IY_&!zNBvp!Afn__+0&y*i^18^U6#4A(Y zT|xY|p8a%l!;Wfuk_|?)<>@JhI-=BcOHl5vL-POcU(5fF@n7-8HHqW{K1-7Rr!UL@I^U^pI`XmS zai9zEWt4qilTG_a;f(R7ve$%P&-jmLmWK`-VF;jyzB3T}pzjQiJuJd|h{QWOT>|KI zti97ea7aIa&d+p6*}(UL)}5@=P0n&iqd@0Ra7Y_L&wuEUmV;jox*b%^M%1tt6>}U? z8&F}ALz)j74k|)^%nam1dKGA!Dabby5zoNU`QBWIbRLW?vrrXaoX>MemqCR&4#_wg zhRtjJ!KF=YI1Kl#;A??BmaKgtZ80q6S6pU5us(c(*f%-0XNUKa5@A(frp@D)}R1T(`OFpCFt?BP>iu*%Q|=r zI%WeDV|3ZF$sskL3B{WoQqV_G{3CFr6Ml6_<)G((Lw?}m?+(d?a#o#lNaI;~=MfR; zNw|OrTOe>A4+ofnix5P{y`V!tw_HMjpsOy!QO7H2f+eg<*K| zH5dT7F*h8NFASV~%OQ;czUsC^+5#$ya1glf9Yh4_@Fxz*(gsId$2m!w47$Ztk_tih zf}RHzLH`C#uY><~n*%v-v{2CVpzA>eA4xg@x~je;33E}WjU=frs1PPeV?Yx?XM_4i zOVSz8|w_8SU>E8J_p%w$XQdg+RVWH9FTo3=LNYQfI{#HHV9Zr8Y&>`j1tY0AT zz+VXF3TUkWs7^J~VF5xHesUwaH9&~t3hjRe2yeObJ_2nUDMa%735`7it8ycS8r&|L zA1U;}zwZATDRjoa;uNA4tEwU@%fkQ`yoD zO(CIExxSZ2NloJ=Ek7hcrM&bh_rC4}tX^LO+NVTJSZ7GTPKF9Qg2Y zIvpI36&mvieQC8gl)Z`$j1$86paFDN99+6e5621qsIx!ggsytiD2G&?*76b>(LT*k zA=Ae=D}=6VCM0po>7UJnKGl0Ra!3~&!sVUy=sLaNLwh$zEgYb8n!~HQ>GbF3(EJ;9 zkB85r2GL>hD0eyiI$mhUj~YVn#|xdfmo&Zw0x!Hxr?!BwNcXi6I_rZn98wJ$%nQD> zeoG-50q@pQXstJ4SS?{@`O-fd2p)7-OCiYXGNxd?8UtU@aTCy>XlYBqpWFp&d`(*O zxWR`8CkWAc&xHGF~Gu@I=QvKBe!$nSXU|c%HT_1HG|y+i9(P* zd5c5ZKvP`c%&SBp+H2!C|IA;Qt<$ZiDXmZ|yXeqXsFhVm98xR#h(%>!B-H%Y3hpmB z`Oj<%a?p{fTWi6;!y2%1-Iy;*i#Y(6fbqJGYZZ0|SaXck&M+i{bpu!ou%5h*Ru2<` z=)BfKeSI4Y*l*F;HbO8x*;;78r~gUsv=-W(2D9#DV4u*epD{-#CJ8=hd%co`#!=a&Rc5dT&L-sXzyvir8*{>ejRWRS zd%Z%(K9nTH^KJg7&ys}R+=+`hD}u)2n28K?UK1I-tl`za6IKF@ z|KXv#&;N(l>6YkAcqASg@*wm?(69B-l?z+&yuMnkuRL@+s`b|C@~YR~?V(#y{Wjp2 zHT-|{(0$?_dcs3@$la*Z{pIdg>7kR{Loa*iHrDjm?xEXRQ@_STx3^{u>Y*#FTeB`2 zV$C`$P+-l_Wgfa^9!jAJ7b)3JTT`a3thb=Ds1LGOX<5G3k&&V4_f)LFp(eMgHEss!?^YI7aN+}c3Nta zHH}yx`0&HK(5VZAb5PYQAF3W=L5%6)Dvel()ONaWq3{PXE?I=s85+D8>UClElZ%BG zoS!aEvDi7C?Loh(&DTV)_ge}bxpc%*6fi!9PF;!+49}-qmtu7HJV_rT4W+@$ga?=z zYApv@Pq`JaZ87y*A-u)Mgwm-ikjFEUZe79hgwe-H!@A&=XyVDQ=({V0Vf^^6_LD1x z3=X+tR|$XA^K76PgviMsHw;|05yJ6(X~=5f93OO@R<0Hnas_tlCjx$E<|-Y#2BYgk zx|)%z_7iJ_K#n_OFZ~P?C7s+vlj4T>FX71>CAErrbth5+BroPTeR>M4m?*<=}qw1vJG((%PF?{miAGZ$bfQ z=r5a~F*%X)n`Mm`He>mLNxX8i@BuS&ix6FJd%WUXgxWlYy`?mdo;~SzTVyM*Z4n;u zo&ogim#FjPCA7;{7WGzis!Wg3ty_V|ETbV`q3q!^>35*@g5nKwsNPXSwquNd+;v7f)<}ZKi`e&8y-ygJ;Ekr{&Ej=6wqFFwCOW+hh6w1E_s3C zFV08g&nPUWSzX37QVhmXq5PSy;z6I${dR delta 19318 zcmb_^2V4}__wUZY-W>}?1zZHAxJZ+xqM)K+iHk0Z*szz_MX`WgS49k)l}LEcMKtsixVsMH8_cA96&+IF9L3Av`B9w2l>cfp>(|Qo)sJx3dOAr?_z1sK zEXzOt2v0dBSb-Q zAK~lo%6bw$!cznpH-3Z{%683~l#dwsUb2CqAK`^nvZ65`;To+hKj|YpUokx6BYa0g zSw3IFYiTt0Hqq>^6XqVlJ-kjd#yqEt-4XqIv+T97-8loQ9hq_2*+cp4p-jMR7OunA z>r5iMsT%&i&Ll8RA?yYg*9XJI8%#XA^T}sF-eB;{;@5%y@rGkFjtS-OK5$I+XDvrc=o5XPNP zo2tF+$T&mf4DCkUnoAmu(V~a(bF{6Y-c0RJ%xpL|Q~QQp^gA4!rA=jqg41lRKNA6w zv$bCA!b`kqwzh)NJ)-rjdKfxS>jA^(YTvRKFY*KCXHO1WTAc&CY6Jgfg*Jy}%a+32RoZ>b0%*Qk8_X<$L95Z-wUAB8 z75I*l-Tb-LTEU6kJ0A{g)h=d|Az_;~it&cY+q8$6A$-JlS|^6-18ugW@JJX=$z(ov zyH?LKwfPM{Xm2xUc)?HF9s$fl{^EJ?#yq zdnZ~t8caEbE?@kv)o7CPI_Zs;MTYY@fTQ1-Ea`?=qh+9hD{VL`!o_!yG^~x*J%;^w zC7x)VX)w7gTCIgt57H>9yl-J~d7T8ku_Pgw3K~lm8P?Ma{1~mXU5ysGo-i1V8m+&q zr#KpqGFnauf14{sqovrY>1dq*1s=q|t6)4J${$~s?0l3>wmeG;DKT0X8FH~?3b|uE zTF#o{%3ZQIVmZljBkAZ9)?~S3j6O-xg-bD>UZ?8p@@kjG z*Bmq&Ye&PR>~d$9&)Pa_GzZSj#i;-1s7rQK02XPJ2;2U^rm!40q9A9q&TzH2tq=+8 z^rU`b6lxYW#-iVOa||ibZ;X~dO_s7GOQCQxzEZ@bO|sk;j`gk-lPuUWCK|#X!+i9} z7+03%S=x3KywQ^;?x$ExrP_f|7Hjo^mY$@!8T;2F!z!$iY@N#`x1Q^onVOt8+BMk{ zM59bGSu+i3G&!!=vPs!C*C~D`3wnb<^aY`LTWVQHExRUJy1SaJ*kO}%-ej6kl*x*9 z*;gx&k#9^SA3e?q^OR~D0 zaAc$uCR#fC8Lfxpz>q&9$zn?iIZ=!~zUb(TI-P@)N;=142pVHZNoJIBf3@6i&BnMA zm+aOUyD)=lmP}1(Ks?QUNk?sLl4VFRextCpccZ~rBZfM|q~tFtWm-V@S$AF*Mnd#U+PjVdqUZU?a}KL6sxAEc%Ag zXo}0H3!_?~d2D7JZ3#w;U^ZsI!$i_Hb)~6Hz*H8Y5z5EfCMsky_!)DqV>H?Cn5+gA zjB?pm1GCw16#b};X`~IGNeA(t!ughR{;+ySgI;PCnnId7I-y9dCVWpcW(~%#H6HkW zyZkY^{9KkKo;6vbsOzrAxMxxi&(sPowCf=@1H-|hMk%I9d7$ZCG^kP(1py5|?ul2% z1w~iU(tyN)iGvacCk`2O{D>=-h0kJWM>%24i}@{`(U_&kO_oZNa1$l%^Z=;-8LYq;_hG?Ig@?jxp!OA3;p;M?V zGM8*O_W;eH`#FlW9BQp|>8D1E`FQErQtT_oldSDbagVa}rTwu`+PgHg^b=_|kQ*Vh zqp%DeDvhU19H+-io6$cJ^iK%>ksAwpq#w2fw$4ZtPFVCbOIwOP_c+biRCJ$aCq<}1 z+aE`LG&+$V&{JN2g5_3?puN-D+L(PDoBnnt8SvP<1vG6)wlK@#T0>ImkxOH@woQtw za#>`;u|>OdW^?${mz<8?){OGnCB@m^^Ohs_xE%_btOf)1lB0$=T5V=C_`{F1=)A)Z zvulb*!VP3;Yf6|TZ(-iapxZEy;t^p*dU8_jB>secy;+|oaPNPw(c^4`jm5-Tt zX|&E$GVv32=wF#w3p)cy6dM)+_XEgcwox>UZ$#!WIUqD57IwgIFx5cf*r;gu$w2mo z&TJy-Hd<~86EOs#pA_J6VOUey@dFYq5fXoeyg(9LXKMuYtpsyG7yG{j9xexxBV+h* zs%9!CptU-Zjtfi}RZ=2^AWgD5Vxl?(3$-zIG&OFLrQ1TfL`#HMr$y1lH$kJs#~JfJ z(aXK0KPty04tU%n9I6|WBt1c(^uckHbvIqx=40;$Br_UQ zo_7B*MstA2iMLW{iDof3$QoB(K2kEaEB&PRaFqy;F41*CNB#fzs{1D>b z=z1_lW_2?db1uxQC9hP14jU$!_E}u)gju0*DTD;-s`_J}9UQzelmyfY4OOz}9}3~2 zB!M+|fs9b{mL(xDCXB>-H^HoQ%Ny}playVgM^$u2b(Qsf35UW+9P^fc6GoOZ%ssG% zlXlDtI1^5y*_6g$3nw$NGtY=1J{~PYD#h-3&3Rm89E~|oGdZK>nXnPQk09@u6Rf|C92(*WvheR9kO~0d7Q+QKp%7&}a7m>g+XT%1AKAox^ePypa@va5bnBdy2fZ zROSU-dFX6^mP%M>OM@3Dg(L7q6q(Bo3j(KT5{|>Gbu?LmBjIE;38*<-GI3kz(wKi0 zO%|}sTbS0Iw5sckI+C(WFazj2CReb+vF79!@v4n}@vtR^1hZ|u;Cu|pX5$({?-pdf z!`p)*Y>Od{@qXi63|Yj~fL<-g0;U8=EXijs1H_U*<~$rm613e@+Q^ycBs0%hZatIb zo^YYz2MyTC3u?rX@ES+DqEUaW0cHaUXD4~X#5nRx(C+$jCcGM8A6e;53&(|TePr;B zL`#HuJfyTDo$6e{>K_i-N3f+8InwEo{=W^`Xb-BSMCgZ99kOk_WD6Z7OSgr!5-kx7 z`(S%(vYHLiL#H<6DEmo$;NnRW-DkLj3ys{Nbv#LC9)KmDyiID+4zVlss4FLM_Qm$f z!i_^~N(1QRwidkgo{YS19)mRrdNwbp+cy=HG!O;26*c1`ON0iSjtY0Nb^)PZ#N85_=MZVogAqal+a zI7vL0wY~V=;G@MA5!z`Y_;n;BYPyQJDJ$!HN_YlebR^NA~XB>EalIBLb@`{OK4yuOPTwy-AE4Vg_<<= z*e8Wscolu2^fc`TsoP*q645hPVMP*Y!EAtElSmX(2-Qg>y6I0k+AVGi9#}#6vNkOo z7gjsTV3|Zqgqd4Kag=?u!h8d1!lu-c*PhixU3&^{;ht<}Em_3^N!4xPnnX*4VOylN zr)~-N?~Z(hz?DE!(B%sH@t=#2zm{q%5t5LqJ-G>u61{9LnYt}3k!XqFwpm(y;O7`z zKq4IF(H)|bMmO^a9wy`D`wr@MBcEZ~zV1e5GiSh~J1*U0Ho~ax#19WFKI=}}FyBE@ zcTCVpc-@^uhwfwKTr_mV1kq@z=?zNODkNGWEZ885`S3iM__J#?P_G-V*tk zJH8Q2??IATZoOoLUCcsmchZ_oB2e6&M6zQTc!TCb`-l~&`nFJj_JlYzM(rd@@!u9! zD|zr=N9_~_5Pyf(whDLVYuTR%y%WIU0 zeiMBG1)9d;@c$9=dy|>EGqHF*_#L?RAtM^c(j3u#XNsdU_^E0P1&hISzaA^E|9h(| zAfdOmzRMcQ%sXTVraP6vl~CA+^uk=b_ay;1tH<^wZEIXcXCF_-+bMY73-91tRe^hN z5>;n^1yzeTpLmCo)j&wH!C!qzD|SL9_@@y)TdxXQr;(6k;avqrX`|~XZVcM~dt-10 znWfiopAF5>|KCufaFtrSh_RNsW6J5iHBE49ChuF9z|J({;gIuAa^Bs;D2zu1vdr&I zWtk)qUZ;^n9l-_^>b!&aek9TFY^59}JsJ=;ytUT}QtGUOZ~NhDaubUBksXWxS^dc; zY&3zx{mGNM9UkILQDI(WaK(F8+-UT4gI@-ae)_AgD{zR2(w;5Raue@j=pA|8chF=Y z(VNpS7Cg^c0UWzRa1GI{O6U-na0^U3y>pRQQ#peJek6U3anT445ILN3e2c* z5Y6B+1vZ*%+Lgyq@!1LuQ_rcmc)dDR{1`WjBRKD{zRaWetT>6u7PmFQ;&v0{2tlsT2-S;Pxs!9$~XkOTibb z_(&@L>V+KsX%!wy;hPHV>}+?u4~0)CFsH(uDg3hnH&fx36keskpE~2sz1*ISDZW6# zSF7Tl6dtF*=Tx{Zg?lP6=VEuIHig?La9b6wLE%6J&QxJ7h0V?izFoyDq7Z*uCWn7t zg`ZRSjskn)EGRoJQ23MrcUR#@6yBx44^{XEh1V!>H-ycymXj1;q~I-8@d64@P+*rj zcAGy@xQ_y7sPK9Uw^QIS6<$K&U z!oMo;cPiYD!h02XfeMFF_*(@YM`1HPo{I^f_-6|Kr7B*Z!jlwutqQwQxSs-NsxV98 z_6j^mh2KOX9HziyRJe@7ZqLnfoWoW8A;qho$?!lGzCz)L3fxD9Pg3}t0uNK+0t)X} z;4Uis1BEvza5D)u)9^b{&p%i2d8&9Eg{La;Xcd+kZlD6^s&E#S?xetzRroN4BNX_5 zRCpJK>p!b7OOyW+6}M4bqlka4!Y?TN_#Zhd*(%(I4xtMQ+*9qgkre(#fxlH@KMHSB zU`|$I#_(M!o~Pg@RXI!HX$t(_!z${^5Cx7@rKNFbRA9Xd-=ose3Y@0GS1H^8VKZI( zQdRsniaRLc?_H6G=aZ*$`0rCL4bMLm*twp4c-5Q;JLtcs5_Dv*cLA#b|m?ZsU;Ujy#Q_$Y0pl1 z3PVN{L&Jf8SK!TGku|skH#XGgXY~A3e-M{>4ZX7&MW(ZzpMq{Q2@5}nvUD40{AkDW z0(<4-4X^On)>4A^dGrKo$P-CDlLV7T5pU)OY#2>aS`TI|pU#?WL88CuhomMb!x^o?QV z7&6|cRG=OmGL+NP8G7{fejDZY4>TN02C&VZ!R)bQc+DZDD2m(h&X3^HSQ5`}djyH& z3EmEmBMtHPq5n9%lkcQp-;N_4m{Ld>PduP}9O=yD(d$8MXgY~!&VrbZcfkw5G=Z4d<`3ZO33y9~H!2fIcLt!vL`1vXheZ<+t%mbh zWOjg02Ieo1jk7vl35MSCy^Mu8yq)@bYb;(Vc#UuoE?52 zo=n0UzwOXyGU>$Zfw7bEY_ZuLSTh;xwt4^$CX+DM=>gQ4f~T6tq5l-JrI}kbJ;AS4 z&<7j#Aa2$11|dGVA@-_^f6*M)DoFZ(VWo_N&}=G+^x9QLD{}v;xQ6m4{{?fW5+Cy{ zMWFu&3Vx#kaj5xmtbpxGpG%#XABL^OJod-?0Nez3RB8?Spq7I#)^eb=s;VR*Uqxu( z2SQz`(5(k3bVU*Z&rH&iopTdXGqJBa+=Mfkcu)xNCX<+&e0~Fi!<& z7iO^>d>o)PzLuICdq~3ujHVy~W0AUypu_l#dCLt{DLr8CIvOuZ`(hOjsY+}YYp=gB z(J}3**M-8W>#`Fx{X#9tSM8HmR5?B-!>4#Cxy)0NoOm4)W)uCWOOo61J8EEd%CFGc z@%!j9se-t=b9yHugp@8G8l=vxzs`>J{9WjZGYJ zZRH$da0$NpfwSYU@XhAnu_``_u)cS&RC-+8tDLGS9G z2@f$6JY(sI_uzecrWh?3P1cyy-CG$q?B9vjyi{C;yu_`B0;pD}ipo}P$;h{2aQARhFQH`F-M%?NnLml|S1C`N< zGKQ!%&ZrFY63PiwIlENOYRXV&7h$!^xI!5zYK{3S!v$NTV6e&Sk!zyR@zpHFg8D*5wOEqFu8R2wHy>GuMDkGUP-e<6%%CJ&KS4kD#mf`?VIR%u{ zLp2hlGTu^#r^;|v8Re7_r!rpSIKdui?TL?eVm!R09=QQ~u!Pk{te1@CDcHE4t_Uua z!I^eJH^lbI*hm>Gu5|4XRb1J>5IL+jqrpRwu6F4eh<$<>P7p2MOV3fHC&|)2f54Xu zh+CUAa#5oSX47X3itH7cdH<5^@K^cBLgl_blJZ~S?gD&CV*-}Y{LiuwI+nfay@nrT z=Cb$93L6`T;_3xRlWaf*D_A%QG*S~_6swq#^hg8aV-+eR+180nU%6UJ~9!a^R z@m!{i0jjRED#H_>s}>AX8GBX6aLQ<^GS;e$Ih5h2GO|^Mc@gEjpMAzjoVK(bPNKNd zl0Tsk=D~|WR-DBWelT?St?_P$~aCL-Bkvtj7rLgQ5o~hDyKd^ zRxRkHaz?6*j+F7fOPi2E*Y3aVyO4pD|EXHBiCS?hWwcWn^;E`L%8BwaQ&N-`Ec45ySktBOlwra-QZ~UX|sC)q@|qpM$u?B!S^!)?(7N&Q@#!qs4)~ zAA|2CxN5)PFE7TM12+Cwh|VSM>?fzen2V1eRzXfKNr)|bg?=nMg=#JKCumw>${D&@ zcq{ZjhnofUWgNO;NV4EtTJMDD6sXQ6huJCJ;gE&gV0(Q7Uw@9T3XDAre|%0_u!#&f zSxJETuLshuD#`kV5=VRNzxbqk7!nL;Fxrv!X@@_S8z9TY$#N~I9KN!pT_MY1LNa6l zy)57^Is8bNd(=kHu7#{aHhhnxFZ^UBu?|~KRKU!)+JNB1lNBPqt@T!j#S7)3{{H`X zsPA z9>Y4GfN@`te^|F-=)aVt)%;rW2w!f%XY5OHe;;@h9xo+JS+^pXl}`@VUNw|HnW77k zMk7Byn^Xi7za)D@4jhphRC;v-Z*+v!a%{o|>IB|?R!9^dFo9_qX~2?WFlHHP<#4*V z0#0-%5zJ$_w2Xu_yImlQcRq@P_}Fn;I4+!)y%GWx9oq-vQ#`fPKz<;JsjI^ZA3j&E ze*~s~MG89AFQk5b{4U6_!&GyLU_q)rj!Qo#o6D5U-WGn9Xo=8k5X@Rmp0jn2^XV(d ze1^&ApROb|7$S)2hz4(%BxJ80xr6h+glvVc zzah<-XK?l#;_3O?mC+1;SsfgB=E?A~xxte&4Rr_}g#=db`bWL7IUww5z{kS5A z_gu#x-$*7f_&P)MCei~>2C_GiKsNmw*uIH$WNh$k6Y*pXN1^6sY^U>(yqWmaU5T5M zEOvH*3*Wl9JjBr;Y~bf?CJz}l^m`bw70)PgV8>Qch6e~6w~-)ryd#|1MpBt`e8cZZ zONLnreYWG#Ng-ryCvWhvWcv4{CmX(!Kl(jZbqLll;PNA>atIMMF!6izDUToe6P|c5 zx$xu9#O7eBQt+qyc2Zb?Y+%$JXIRxw_(f+4)REWiubvhbqC%W)+d{Tw|6qw~Y!ce<-W`<9Laz1k}InKHk zTNDLD@p*fib{}-!PvYX+ywPBmp9TjmGNfQG{~lhJ6+C&5As??DG21>e|9vaqyTQnk z8Y1xx9NtfUVGXbNSqJd^h3&JL|KS&WcJJo&oVLo~`<QB zHrzZ$ykKhqdC8_OlneY3EVN%B7fK4rc1-z{qht(QcfO3hJxU(1bGm74c(P0oE3if1D2DL(5LlPid~>3jj`#pjw01RfxmyR&yEJ zA~)YgbHH^sKknvf5?x?X@=Bkx6zW z17D81cp6_ey1_GN$SRg?Iu+LZhQSWI3x|KhHvIH9%sNX#L-yQ4^~dC-&VdRs(Na9B zy~VLvaivE`LkhO3b{R?t!vKCeOSVRDTZtkr`-+MWhMp|0?02-dJVHBT3XUq`VjYO@ zEnOp?N$kM0UHSMreH z%GX{y5@nBbs~u7EHFEUF`5IuCQS&v^j;Q%+hp$39TqlW!>wr9xjGV^yC zOB`G=kmA>+PPgy@J3=951;UwW{6(00odhw*z~u(eiT-7yah?F&Kaym5PkHPeNWMo7_ zw$fUL_Smq^SRs-Ev+xBW1I{$syW_BuEn^P8*jSVBcesHEbsyxZ)fYKem`Q zQgLu#ae8rec}el>$-lG3b2GGMh$D5fQ2xce!GR~!i;iZMpF+&NcM-*KUOW8+i$0^I zB2I&6%8RQfzogXzrxfu^{w4_yZ1Z0H*{7tCal3UG6;=+uKVEj)HOXSm@Bwe|O6Sz%XrKK{?5S#+5ipdc)2N z;tfwLNr`{qPPBV;*5Ff^Ykb8W*F>$)Cz7`~D5nek#=#%&@I~YeFsBL^!ttE|sz{<+ z{tnsO52nE!cv6My&NFbXCX;=uf5tT3n;V?w{?>lwQ)a&kEQN2Y$xyc8n3XotJoYN$ z*bm}&$TUVNhZoA_z})WizdsiF7Annt60=yH;l7hkw~^TnY)pIpF{69x5Y@O1_1o2H zo)Fy$C)O_l&eqd~v;L`2T~FuBYEq%0o31|I zth8{`Mc{^bq?@i?v#V=RY5Pu^uPA)1ljbih#OT)Y zSKM?R8RjT>d!U{R(A7idhoy86U8~@gfTBm|WsKfK|7Kjm)gd^;sSwXshVT3<_;#7R z2$%uRdg%Pw2>@jtx}j{X)zGuP?vt?lRIcer+zQz-M->X!wix;M8*x8eRcK z^>w3|U3{2c*Mngv1;ac~T~jvQL}T^QnpvTX2`+i+zG*Q&6w~s}^`}J(dpo8_I2PR- zq=}in60z5>e?$Cw(ZZ3A10xI+4>u!DFXC6i)&{yL*BmSle~}!VdWNDtE8$TCUHh6- z6{*0_`DR|ac!v1_CVT5Vm=%!gt=rIfcMvLHas44p)@RF5!9^U256g>xn|#@^xOnnQ z2V}k}8s1x@{q8G;`=-3;*dWJIjz!0k9jBoyy*2Sa!XzJE?|RcwM3VhNxp3bMSABF- zYxSUI6bv8p72l_!&XcK=g|?(ggU%A3T4I3tzPiv(Q$DE>aUf)Le_mWR^moi2ml62W z7if6b1Wt29Du=7(zoc@ETK-2W$NvP+e06)+xTXANKbHwx6_0vJlg`tU;{Pr;2-z+2WD(7P`Yi+MM1dmZM*pzuv#bBIP6r;i){#daPZZ_AM8h;wa;6ZN8f=D&Z<|2xKi#SeU>a^a`2X;= z*mbm1E|w}Z8)<5}O`L|`qxijq-$(eZU11Zw@Y@l;7W|IFFLoUb+|ASZ@?xHDawBur zd{O+reeWy~#sAZn*8h-i*H?x0==Bq%wHL~iUVk8)^e^L-@v*WGgg>nKuWMQd1`pGD z!;xJ&!uQ{$d*Ncv9cL4JjnruNBb|f$@2eAR;;%@#44asVcq~%485&LbB%3$_DL2I? zZbF(i)g~@S{tTo$kdDd3tc^mNWfR*V&7EcwbCGUCDxkbD2kT*3KNtBkvEDqJSTqXH z&tvD?#H+{%UT71SAR{f?Cf-1ryT~RQMx$dnHn9nwQC?ka6Vs6f=i0&}W(4)@N1ACyfwXHj@eoyj)QA;h|Fnq%kp^E! zgGlu^Fj86(sc9irylE5HB0YHvGlg{9ZOjzT#N~I;0hAN&+eAG&SpJty9D}(2kxkr& zbc}!jBEIS|W(3RIUf4u)8$9PSGNL#GX|SUx79ovAdKKvyq_2=(t&9JAmxFR1qPPTU z8q)PhbCDiGs`n8^ZZWoLV^QpdG#BX@r2COBLb@tK6wf0y#)zUf&hBYQqmbsth-T4* z7yFY$aTU_?9-{aFDVHjWhw@#z2sAQ_#`Rv5YhU>6ocOqL_{sGp35-e5AQZms5dx=m^qP zIMiaTSdpJltb3cvG=Y(N&X-+M0~YJKhK&z7*+gFreiEHsv)O*m9A296+p(ucQ;57r zl-C(-Cv+ZgN6$s!KfTxSf6KuhE~B;;Wr(h zFQ$3g#K-XaV_jpY*MRfa4)n2!ky4o-(tsPuXw!XdVjfKX9R;3wasEsK)bU2uMU7yv zH`j$N4}|UBTvMhaf76>A;>51gLEBI+6yJ%7cH-P&UMT0xG>5!Ut~=uhH$%D3xbY4Q z_%g0|r(Jp$&2 za~&8B91Z8%F_VCa;JVk^(ZD8N!FU|KeBtM7Is=#^Bu6(ypo}ZrjlddcH4^ny`M|VD zRNDlKBe~9OkuNw$p|w|jFgS|)gk9nfTcWsB2V;Lc^s2*kf`n+!G4x50O*~1Hh97zd zdt*E{nD0aq)~B+-%16 ztaU!nCjQw7Rqt*9>$F^bNNtX7&<1jvV^GWc!rA7ixfeLaV8{>p!QdFQ>jpo@aP8O! z1K^(+?h|GX#I(S?=MINiEl_w29B9FH)~+9k>lOI3oF05ya*>$t2CPx-}Q?! zH5xxiY=y1T90s+*R?#1`iLIcFW~E;!s=3q(!>^6g@rP!YWZ^J|8m&364&KO`UxUUX zcbG%S%Dsr|lH+^qJYsvUyjPKctOtngKrDsTK&>Fo4?b(n`DhQ}v^@l(+i?DHx;593 zy?O)wZq2pFwh3>8Jv#v=x51VhgOhO{_@BdZdLKnSf51t(zidwGplfl>El2EYn0yvj z=(u>UKK8vH@myH=E1b9Ok4quGqp%x1>v4`@ud)AHjf)bp{a0%2ze=P03XR783p92)zw@#_`)|+Ke{076 z+cI`Jz>++fBJez#(Gi-N~iNzd?^QZH1e)XImEqiQV%=Y29smnU!sP|lk zIy5kB85(#V3A2`A3hpd{?aOd-&p!n(u#8Imzv7958QH++g;37yk50ZWM#HqgQcP8sz)P4q@h`PYs;7 z>S79?_k@7e++}wCU8r2meZeH~majSdNtdB8b}i1XlVCL^L-|u{IbViJ;U9m4>m*yK zhgR#*XabB~$1TOMp>iGf4@y5?&qcE5zk~)GFmF*%wgDa2K#h&qE*k#SMh>5=xB|0@ zo6fFp53@FL)38q2CMmfKHlwB;Ay9V}Uju-rd5#p5m zGwnV6fuA`YgOMHG!EL2Z?!n_dims; tensor.type = cur->type; tensor.ne.resize(n_dims); - memcpy(tensor.ne.data(), &cur->ne[0], sizeof(tensor.ne[0]) * n_dims); + for (uint32_t j = 0; j < n_dims; ++j) { + tensor.ne[j] = cur->ne[j]; + } + if (n_dims < 1 || n_dims > 2) { throw std::runtime_error(format("llama.cpp: tensor '%s' should not be %u-dimensional", name, n_dims)); } From 4c0f64e30271a21f1d779ba0506a58d27c9858b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Thu, 10 Aug 2023 18:07:41 +0300 Subject: [PATCH 087/242] rm binary commited by mistake --- gguf-llama-simple | Bin 607488 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 gguf-llama-simple diff --git a/gguf-llama-simple b/gguf-llama-simple deleted file mode 100644 index bfa61de57dbc77cfb6c180b20a23a46dfebaf99f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 607488 zcmeFa3tUxI7C(N#i{h($eHE3}t*Edk_=L>5$is8>qNHS)7AS&H2m}{J^MPF`Pq#Z! z*9gNZOC(84#rxRv9KHCBhP2;6UNbEf@1qiJH}DqQazoQ>D=#jxpdZhu^(1l7eX%!l z{@D4Usa=(qmh!7yCRESO>UCpk$2Iz~v)@4(rhns}OZP?ogSXF^77pY$XsIUq>v12K zm|EaT6192KRP}jga_Q*93(`_9T}isCuwc&UkyjPYy`r$7sJ!Beiulo2j2?MqS@D&_ znLSYX#5-Z~4XgmNFI5ye{U_vzht<1sg|s3}swC_&{C^z(x97e;-&%FfxBj;euH5y= zj@JR1{_HzVl2p-z@FYQu-QguB{2j1^?)ZJc2ZWT&iBfdK?(nftOn2o>hkUM`)P3p5Bis_Djt%-czHF<6V457PnENEAEa$9K9-qCOt z-Ien?468f5-lUvaCiTDGr2J&)x4ZJsGqImVCUXA7g#Vxk?l6)63nq9L`e}FdEHPHEFNSqDO17@IPQ8w=l?neY6Tk7ciGGKg^t-Jl@~J_;?ymm%CiY-4kz0pJJ#R7bxBE@%Q)?oJ zSQC7y3H}=J?%MULiTock>34M|<64->bBCMQ|7j-p7?X0YGLf6zg#Skqyv@WP4lu!! zOv+gey?5t7KQ!S#&!qfUOzi3x6T2FO$!&Mza=Z!u6qEWKGx0wMOzfe`q?{Ed&%MiJ z-g2Eue~dD*hdWK`d9z79=i<5D>1BjTJ)bfu=Qfk^{U#HA9W&AId8m&a|BXM_o7Cr8 z6FCnt;eXGhKI2X5<1>+eACvNzn)tCZg{Om}|iTN8d1->p2yn&9u6 z$g{)bx#LWpTV!H~-Q31 zuXTH+#GPN7J9}e7R@i7i+bENdD02)Jnmt9;!D@RmvNB?`Ql^d0nmTO+ zM1NDp^l9#d+=`ss5_dtdDkRf(o76rbD=lkwSy{onBFJspNGkS3(mPQpC6MD>lEQpd z3|z}`qwolBFB~Ar$Ceir++Lmw#g{rKPs?Id&2Soon2|xXI0aK<$vkPYz z<>byuhjij~mLl{*rkiQ8uyM5sr=$x!5cMWe8@n{R-$Lhmjkokaz1)uoAcVf3#iEr^SSfJ$>q3TNl!rp%uY zNzen}ZRh2>5xLDZm#n!X9FsNGye9Bv(?(@wkoVwjEn~XK9F!u41rsT%0|n}tks;pu zo4Nw2qG&dgCKM7kJc|q*qgY-+AsU@Brl2gV7><+r3+vr%pkShjgBdakcfe)~F{)6f zLOUQ#8r^2kDOeCkgG*UKR!$+jc$Rxn33ar*V$D~~!N6RUl{4F&lh4Ww4n<{)FksK; z9*JBd`9XulhWeX1#iefWiGnyq-H$HKEh|_IwbS@Gw=kW%hW{rt&)OU}T9jkX>@v3| zYwk%)G0M)!cf$=218E$g^R*YGS1 zC?->`X$8zTWKUwm<7!( z_(VNrDz%b%snXn_p9+qz#pUj-;=HWV*+uhmv-0NC__~-I?r&`vJ#>CKn-dCy1twQA zDPhhccWznwqy?JL_0oTnwRST<7aO1s*8jmMI?_0ZY7=?#`6gpcO3>8=r}VsrOyHPW z3vOJPGtJF~a@b@By``I}6y%Q!37QKmES`tKC%6D%h3C=yEa;Z7WT+@`OQ;fBD&^6tLYmJWv7oG|1dCS^zI!e#RLY||5hqnL5B0#{#U5CW zCu&7wQC>>zq8gCh6%|W)3rh>I+(tcT&jsV$1qHZ~H6shvq6G?y=1=0i9O8z27Gr44 zs-Pu}1h&$`g896j^NSbc>OwEeEzFbV6tPJ;(4%8Lk{MlH|W9or5TAnl^ zDJg3NbUwl59G{dm{L0}8!R(k|_R3Lue#Dg{c|J1}caFSrxO%fn#;JE{WPB*|=nyh4 zE`&^wk}@+B$2+s)t{fp`wTEjD483XiaIp-YM?D~lKrQ$!g21VkJOLRy}{oTcl1ZvLpqIJ zjgYKJsci9oIPU4ou2Tv8$-HDMJR88*;Vhj3{y zep4EWe-@P1m(>{|E@=qR)7e#OAGItT*QpMgl!zjnFp?Gmiueh`za+%kw0K_n#fuQh z93sskfL-q)9YoBHNC*Ba_x}-(J6)O!fUvt6ZIg;sy8NsHL~?sei|HbM!tktYwftDn z?lbB7VCk+9ev8207x(b88<3WCK8z*|k{(d`=P`Pq^rTAXGkSnjtI`V?-A{U3rDrp` zuk^l3E6azWygt$wD*fTQ+Ykwil-gCgd^_zp_L2^%^zH0^NDVU?H=|3 zB7i-l^VJ|@806ZZwioQu!%RL_=?azqjDiA4^E4@5rB|`*{iX3L&E*%a%R@WQgom)7 z4)wfPlzPrd!)r77b-Mv?Q}GxBUaOWP8}L{r!GM>j{PsL9 zc}@O~!MvOp170$O*Fg8^?{PhL%uS`2vYYaDMi z;P%HkzSDr$48q0$e*6YJdkg39FyO6QIo@f&JJjb&^FsCBHiVaNGvI4lc{x!AT>6aT zb^~tP#_<>f?l_Z|A8)|zyE(tZfVcWNE*o%%`rLE_ZX3$W&o|Kbt>*R;BTsUhXHR?@lFHYq~cORsGV$8@kj&yo{HNH z_;wYKGT{GIak~NkOvPgi_?IdkYruD_c)S5`S8;~{?@)2sfFD-zbOU}&#WM|fpPRW} zvJH5Yisu{f!75&2z|T?f3Il$bidP!&5h}jYfM28HH3odVimx`{sVct4fKOEMwFZ2O ziq{(O8&$l{fX`C#1_M4<#aj$`k&3q(@P#VA(|}j2c$)#gN5%aHe6@;q81TQUc&7oc zRdMOoQ2XDY;*kb?lZx96_u^D2K)yVcNp+PDlQxF zh%D7^4fyFQo@u}bs(7{mKUc-`4fs_mUSfh*81NA)f29E*qv9(Kc#?|O7;vYGuQuS< ztN0oNK32uo8t_>vUTeVTsd$|McdK}V0k2f?76X32inki@M^t>L0e@b_+YI=dD(*Mn zEh^q&z&};-P6NJ6#ihbf``@SHkp}#*irWmhl+Eob%7FJ(ak~LOUBzPz`1vXxYrrp8 z@puD%wTe3oc&dua27HE!ryKBG70)!_Wh$O+z!OzG-+(Vs@e%{>QSk}`?o;tf1OAAL zuQcF)Rq+}F{52qvCA_++N1@+iAd~)c4UMwUm`ANwsP@ zHUr+O;&uaGtCpYc4z)v#Zcv{~t9g1iOaD>JG2&Xg37?Mt6ceK#t-eOwFO<($h@3x0 zC||_Wi+Mc>+E2cKpQ(XJO9cE00k05ndQVOJsTA-S4Me(9 zz>@^LM!+Wv_-X;q67V$wzD&T^3V4-(*9!O_1iVhbYXm%7)jR1$#Pdz?2BDk>gmPL0 z{6PV474U}ye5ZgvEZ}VdzFNTj0{)1AcL?~S0-h<pvv|j&QX8Qz76u(yISd3b@v%>EcQOS3}(Fo*Drc&$+G^@WFcNlC(y^hY0vu z0Y6>9YX$ra0k0GAC;@K}aM6#o2>4k7f2)9>E#Nx^Tze;)F1HEzIRd|5z|R%%4gsgL z=h{!FfZH{Y(%XY}d%l223it&AZWHhe1w2Z?qXpb9;1>yajDTM(;IRUJiGaro_%H!? z2>7J}E(`c&0-i45mkW5NfL|ft*#ds0faeSNRRUfj;IRT;A>eTWUMb+i1$?D|j}Y)0 z0cRVL>b2DZK1$$UBjBS2e64_w5%5|8j~DPd0Z$O{1_8fXz*_{I&gN@Btpa|H1|r=l z;Nt|mO~9`eaKC_GC*U0d?hx=!0Z$ZgsWfQ+;{`lYz>@{sCg3Ro9wp$Z0&W-Z2?8D? z;Ie?n3b<3i;{|-8fI9?yl7Pzs?h^2H0Z$X~OaY%F;MoG6F5vkBe!YN~2>4V1uMqHQ z0$wTL83Mji!0DX^?Wab-Z`44fs|8%$dSqAE2)MfS$ndoSev?p6t$@!I@HzpXCEyJL zezSnL2>2}m-YVeP0=`qgXA5|nfX@+dzkufmc!z+`74S|0&lPZ~ENK6E0v;*g^90-` z;Q0a`CEx`DZWr)d1w2N;ZxirX0WTErcmba;;0^&V5^!0-iv>Jgz)J)?Q^0Q*@N5Au z74UokFB9+*0e1^{g@Bg}c%^_Z5b%`(zEHqx1iV7PR}1(e0be8Fiv@hGfZrkDwF16G z!0QBjxqvqac%^{12>1#CZx!%61$?J~djz~qz^et^FW`3xc!z-BE#RF3zEZ#?bw1w< zm?Gej0)CHx+XUPz;86m8uYlVH+$Z2M0)C%>#|rrU0v<2ms|4I3;C~eG|7HItf&WS1 ze-ikg1pX(1|4HEgzY_S#dhSu#^FyTUjdI zndqKGS2Fr}qG|dSC}H$dMAM=qkj>~viSA8wI-?&Tx)0F~M&CnpU!r3feJ9cVh_*9& zG12{rwlTVtXbMdRBu3v#w3XK6)-_3LOS)j6Og#g$x4{qrWGbLWO~jU#b1OiMA8n z#^}$8KA-4TM(-f{0-_rjy^ZJ#iLPbzMxrSM7+AyT*NMJ}=o&`9O!UP>S2Fr}qAwx3 zgwanC9Yb_BqaP)D7}4pBet_spiFPph9-=QJI+oFQ5`8(*c1ABI`U;|Lj4mabLU{p+ z(YF$P7115PF#Quvp}RmEqi-TQj_6iKPbHc{b%6#(Pb8W`bb(q%k0*L0(Q6oeHPIB3 z3)C=rIMEb}3sf@tQlcpY7bs!$1w>QmEs)LVvxugUTOgg$1Bs?kTfo8SK15T9EfCA- zaH1)+7O*q==sBP%q!zF-`T)@sN()Gg{+?(Gp#?gAX8I@EL3A6VKO;Jk=vGGWAbLE} z4UFDKbQ00EjNV8zh0FqL82viYDMZ&W`emY1iLPYy^F&V|x`feB5iJv)&FDvob`qV= z=m&_NNVJ2|_Ygga=vYSINwkY-JEIp9okp~c(WOLBCR$?jtwc{Dy5lI*KhYHG3bZl$ zCZew=x|PvWiKft2pn=g7iKdWNpqA0&iOwK;4Wq9nnnGBC8b%K%dOFdSjJ}j;3S|XK z7<~cJ6v7H*Gx{u|DRdP`XY@d#DP$FJFuD)X6siiuGCG`S3Q+~@j6OOPG=-)DHbx&H znnF?miP7H^okeuV5vG5lvx#nF^k+oRCc2f;JBXe`bOWQe5uHPHEu%LQO`)g28b-fP zbS}{~jDDHuJfbTZ{XEh0h%RCDQ$*7M6v$@uqeK@FozCb7h`yC*2cz#H`Zl6t8GR?w zg+$vKy_o3vMB5l$N;HLv0urNdCAygCj!ve3qA4^KXk+wEMBh$yE2F0pO`)Jb1EVJr zO(CE_Eu+U1?IwB+qpv2KLOy{SMh_=?0nwF=zLaPR@dQd3eF4!F+6iPc`YfU;q!UPI z^gyC1loN0;x)0G5!U@DOI-F<<-3086K6*Cjr9|5peSqj?L`#hRo@ff`1Ue2g{S#eD zbQ_~TBYFkVt&HA5^qoXEFnSx&9-?a*y^-iDqSr9`b)u_@uHpQrM@o4$T^H>LNpYS1 zOp|@*)Z-3W`O4o0|15jQ4Lbv*Z^)et{c>jDIhbvp^*nlnBoTj1`C!Ih%lVI>VDL}X z_!o)%G#`rF>r}SiA}hOP&z_F-43{q=?_WJ}OaC7~r(|3U%Iz9biee7dfE-2z`64c$ zB%#wi1XHQW^axC)dUv&1&#R<*Y*JJ3A32NoM;)MK*fCk@kQ?`1CpUKXki)jf+mE>i zqkt>a0wTNG&=hz}?S86X<+xXHuT*~74YFt4qeGCDm2cerAc|$c{q-2O{d1wJEfGG@ zVK?L6koyA*aZ#<$4T(1_n}2LHK-y1L#91EYS!(B6oX`Q;UxsjQiavl4Gkyd^Q~=1rbo z!{)w+Mlg9T!Rn}ME zj2?u-{j--@*mKuVMfc!CRI-;BFMF$^+W^W+a{u^@AAbOXx_a7Ejjei>ki zZt7g<NX)hIo;nsXb6RHbXr1(p}17dccey=mAYG zWqoud9xwZrMCZ$Rpz>KgiOxUc@}82*3()1A8(kuMlcOuB=IJiwM}OM12r1d;_{F8{ zat%8oHwMCG?~t}Zk+`8xVnWN(3sBD=_po|?jl`vVfU0IfN=z$(N~X}N=#}ibYtAxJ}u!p_oXi1s1Y%}P{7aL5%gfoOBN7^QBM>&<{%W~*<^7gd|loK z8J&R&Kt}$n;2>N|VRUBuX(8=gSGl&l|A)!65(d5*xAS`O}Mx+fQ#0u za>!85`9k(t9!DXN?llho8E_=B(g>Ij5*_RU6Z%{Ayk(K>&!hr>&oo{N0 zE3Bg|A+36k`x;axIeP9ypXFJq4r`4QolI7mnvU9VrG2QZY!A7ywArb=i+jsA$L*cs z8@V7=S@Mds`N_otl3%fmZ%%%NwM(x4eA#I-X=NYm&OaR^TUzyAcO*%4>%Y6YCi+~j zNRxb)xw3CoHCnXS%dX0$j{97xjXs$o%T| zLjc~0CvDv7>QP_nCF#zDX7_*;WlWlHo<*+STYg<&0X#f?IT$Ty_75OFWmB>e3)69W zHe2Wk>2h_KJHr1@ShP!-i!zk2aI3ZI8X6lQj5l#1a1|RUA}ZJeBk2>xyvLwc+jjMo z{9{Ob$}jZHcLL#HCNWOk%RNvfxcw=9ixjulf9_(7lvdqkt@;V6JqD!+(4Fmv~}z+a%(`~Si^(2xGV z|AZ&=l0GNt$^OR((b%t2*%yMHo~hwysCX9eyeqY#n2lp>xXGB7E-ODb_o9B}|Nf^4 zNlCIab;d+2|Ej0d)Hxt2Qm%s#+45UflA@YAZrloZlD+9?Tf}X|^YehD*yf~oNvga~ z_9c~o$sso<$<)<|F}*n{o32ql-{mWFG$&QWuxl|9W7TIYk?Y=j_ulIqfMvyVFS|O( zyNXggAFyl9=GW8t^-TRb;dH%{UoYX;EA;Dx)Acp{dJVt6TE9*>+J9pM z)aI$>R~yu;xNRpPil>ENXjLz4VK^1r$*=qQ^$z_y;b>hXJTMlvS#rBGtJm@a}ThSDvbD~PGtLoaL74+Y9-mV-IYCiooL`vzHt3nOZT|Mm{* zs)zks@iglD*Q4*(ko`}C@yp(~Sxwb?!;?k$9k>A{uokNAWq1q~_+h_=-%BmiM9l8};YU_x3CAlbVfaTJPv>~Hj=#zASdKe%{27khIBwVR zdpS-9i04W=?&i4I3D|RIah&xZAL})|KFPq*Q`2Sd9n3|aq0UcT-s}7^bWOp4)m7es z>kipFO_m#X_lP^>PuttoMJBVkv0^|^cFnV?L{9kGEzcyK1orNO2f~l;gq)r&Hb~Ds z$mzY7@~(u#?soqcu(j_|Wek~GtJh+T3%r0|tej2ydP38m%eT-bdzag2I5}G=pXytI z(t`CJi|R(I)qUcJkm`=bq$-wb+lSZoI^1Y@p3;eRN3VI25{7@vicK}-3pam#d%uRT z&0jA%r$K5uvaDf~@cil>)<2NYl~&JDoZ*h!8hDhbM$fJ`*@tD2XLmzNT!YKk{~vgf z80&(~Pc0o`=|#U?y=+atq_#BqZ`?6w$I@XB3|_%V2$zu-g)`YHnbj+nUEF!&9(FNW zVI=Ct$bOG(esalx!Kw5+In_?d+*Ai8uGDO&r@`S$*jQdICp5ZaFzgLOO?3GtTVJzG z!R*Cmxf4?%%OB*fMmeFS+>ePgmN5<~ZY#Ac(t0oRg+95{scdyN26{NV8dH>(L{D2U z*>fbGHT)4Y-09iX8MjrC4^{z|`!;{Q^c>4%S`j4QXD1?gvW-&?PT4t?&8b+Ts_v^q zc^q(XV2uu>bD%*7vZbbD#Z~vU=|BlY-{^398au7k6&Uu?usV!_{K2Nz2ON>ku9oul zL~2P(kHm!5rI#tFy;%RFjDHc;AN-ae>ykYQ+pWGXJQ3KiZ7$4z4{w*l8lm!D?hCa+ zn2GjoZabTwUNpp#Dml9vM4OIA8(kQ6Y0BpyH`k!M`RoJcIRs^c-_A(+y_?q46>0P(r7l#QkC6|CO|8UipNVU>ZB|BjNo zTE>x4T0NO4fJxn@9C9f~{VUK?EV>7I`=5a)bTumdtw=pbB1>QQ_K!q%#gQ%)wlEh@ zU2S}-X3wF|yHFJN4#w?mKRqbFt|o8)ekd{Fy|QP7wk0dO5*0rrI;&lf zDAiPK;Z@Kf+V)1Gw;G?0k$iGkqT-`mqEc8bwLje)@mw}2{It)+d}+aStT67x2OCiR zn=k1l`HOKsg{`i|vPo9H+d#W0va-?t8d){)bL(RJNYV&PwDnb2ba-CxMg*OF<6a5J zt#gS55t@6yL%L!!S&k6>fgxVjgA6ee{Ijr**)R-6vna%I%z;nQMBH#RT(jKHHfFiK zaUU#q9+LWPSd&2-MNp}K#y5*VLUH z5@vf?>l}3vpKn|RE<)Ho#^Z0>P2<2J_Z2X&a{)1a?}8|&_jIRsa%7q}6_ft2Q0>=< z(YwF~=JGF53|}vfN@d0HDd;VM2f&n9BYP)AVf~G&;Fq0##gNO={Ryr!lrhC@BK{cE zUZyoW+Tj~@+ZU*!RsiE7e#^(W)IpbWM*a~dA@@EYH-2w-_lnygA3lU&z>dR*FrLfS z2^eo0tg@%AgUd(G*}_WmB8)f0%FDKFL;h)2SigH%VcS??J*lw$n^4w%++0O=DzW*` zLhyPq`+n+^0|nO9}ky}eN4jlt*Dy$fe1Tc7TArF*)pWoOpihX6UiGOL`^j2rIa z6`nyJcsM#B4=BcdiE8_XBqD)bzKB&`eTj|P^pgJlB|SR2tPZR7L%milChGi`#pvdMZB~4}Jx~gGQ943^yyr}TnFgamLEX{XsbjS(Y7xt3J-YD}5QGIOk*i>7&9l0pv zqRLN4&W@bD93g2~*Vxn;wb;<{#q)zqYZfgAJMsXYSe*W(uIrO16Tug~Of#o2Dbn zlcQr@zAGAL+Hltu^)m;8jkXBTe-o{*H8&k8Y5@13olTv?W+5_sbimn7yJtjA)>m1;XYA_}KKuX6>g zJPd>W7gzxaf?l>Oge+#{xOjNd*JCna$%p)Jeig!z297NeH{CQ?3d?Jbm_upUmWW#@ z-4cx6a!Gi$ zJv@!Ql&xFBQMZ1m@-CDxD=gCK zwWfLJMY?>qTb-V75wvbYAlYS2*Y{?Jou#~_G1+DCz2Pc)uBWHB;xwtp3Q$Cjp_XGBzBzKPN6d4L#A-bGRy z=KuWjuCBNSJT}q#>J*zx8G&G4q;*x2ZRMl^D<=(XI)Zr6uLF_>(XSB*@};vnp$hi3 z*7EUi7vJBAWv#Lk!ywlD+J2Kd`+fB&Ir?}!ZT(Jq=#VBP@V!oab*esg0Br%lL)YkB zf8bny=3HZmYdE;#&>0u{!(hw)AO6Jx3c7!!SdQ+evQQOxu^3^zMFpKe{Ri6we>R5k zE&m!eZ0SM`X(!<*8J902L!6o8#x_Kvm`424Nki?`V~%Z*64Md9!%3b;(w0Qb2a*DPOsPK$2o1G zI41Ew%;^k`zMs?2XtbBp2Q|8y)5)~Hq5GF}dX+{m;`APkF5~pbo+|%*PFHF4JWhY4 z(X%-n-Am=4$!WJn-@xh38a;*6gCkXbC#UCXbP}i6YxFoyTWEDb^&i9O42_QC^fMZL z8K)0w^hKOb?xWs+9;a7n^qHLAqtUcw3i}$_SLN@|=_-xx#p#bUI*ilN{Z#&+;aG6L zTciKY>CGDbBc})VSNXr?^jwYpg463Y`Xf$TPE+~c;dF*ZZ{+kd8ePxngBo4S>13;V z|6e)1N~53P^d60Vh|?o!(}ms7jXF6g@J;LHNFu*QlZjsO~u0+&z@|D^zZgs={ZkOK61$2tq; zdQ&cfaDexZU`ciHa<@!;MHlgeK zQ*IsQPNUqjl%s90z@I2b&OdNJ<=_mZz}=J^NV%ny8$`J>$_=Jm0p*5JE{k%fQ|<=J zu``U5D2D-53M5kQOv;U++*y>nf^uh5?n259rQDg6JBM;s%AHHO2+EyDxu5Z?&Q7`g zlslht-%;)Y%6&n(3n}*@<)SIKm2wwR?rq9lOu1JmcM0WQpj-^)o}?TtQv(lEj@H5f zALTBioQHCk<5&BVx+^Ftue*|xqPnXnnO7G}Np@WvB{S-VQ!=$~1SON|MpBYoH;R&R zb)zX6T{ni3tLow@iK$DVaInCFpxuTY{O{Y|K-kJ z6733A-eHsIyv7}o{&hRMx-bB>qz{&6Qb;TrH+ZMm27hP8*sHOSsltVM6PVS6eZ+cRPM`po+3gaKG%>Eoee<-!meD}YGx zy6o*`SdT7@!ag!W?p1f9Js6w6f1ifv{vJ$c+b^n#+u=O?lQUtX(>h_Jwz6ak&D0hd zD`MQ}PV+2_lFIjEb%Hhcxf8v~XT?nRW}GEs8@_d-bzSmVHugJ;{kD(y4nE6?1#N3u z&d~d@_;mIDj255ee~jCac=)IGRkG3|))&F@_P#>5lip!V+2eE;bgx`KK*C>A0mFoN;$4Uu#<;vU#v=OXN@%s-#a-mMPY_wc4CNdg z&cL)5HBz?o2}i>Y>_NPO?hZfm5#~;ly_TKO2c|K3kkRCv!H=@~@1XiytNuZZk2nt> zMe7<8Jx9W=K7!gSDc-Nvv3eGwM_3O=(4R}BMY@aDlC#3?u7nS*chivS@-B}|^9=@i zlijHdJ;SLapAk9UXBpdcZy7};T-M~yH02Bzf{>(e zIcyUeTt3j{dyr<|r~t_#&6>PX8PS|XUQ9|>CPgCl9WkpJ8?8y?h!6^2o@ArPP$rgs zrPHqx{PNWB>z=hpUDyOv$yPP(P*aJXhS=R7$dM!~??Vi>(kPeD67C$CE3Sc(jJP&RhQ>KP`zq5C4wN@K6FRKbhuI=8;`VoXN)x^PcFM}6SlOF|VAhah z1m%n*JJTaR@+0!$V`{|5x~>t^b&5VDMX^nIWbDYuT@walPR{nSXe#auYn7D=F@hm} zi1Q7BQXHM2Zg)U8*?T_#o~&$3WFsNeH^6%5bFBGOv8DE*?D-lJNebJ%i9u3?bDVaSG>!XA=e^-s~duzU;mcAz*zGU78u@E+!D4C z>!j5@GTVkkcL{%cpDjOv{uaT-!8kjxi^Wo5(Egb?r#2(F|H1Y<7z=hahz~A-4BtP3 z4%_J)8oK`u;dkK}8r=Ra`lILUrekz+h25md7yL223MilPyKwRc#;^m=A4r0IF5{;gW_1&%wG~v)& zS&lNXV4OTKapmLz*!;C&dl6f}sWAwg;!MwEl+vtxsf9*qW_0{{N7NiqNzaji*1Ky_ zS0@hic$eE@ic6xS*l!eu_T#Kl8cw#wr{Q#A3)RIpq!~sh=OA_&w!Lc?j`#dwCho_P zv_DKk+H~~NElOvyvfs1oykzBoXLlRSGpfbuTbLc!)qcLgzsNXg7Pc=f>_=>aTdM}) zekdo=bIe{kijCK7)AX)2JZa!b8zuejg0q0%-dO%GCpMNaq^~FY!)dhvmb+nqX$f1( z545kL+Pww!QSE5SMzzyoLt6ZY_K=k$tghGL0G8)lM8lRYYDiWNd%o+H!u(KYnr}*W zO5Bd)>hJ8@m=+d5{hbNhtX0$S1XMn3seHoFt&z8*@@K zn3GDL!JJg`4CbUPGn}4H*eL$8{9khQ4)?=2m+Dl0!q#u;QfL=fnc^WAl1G6D4tXQJut`@Y|VSRO< z?CHcV+LGu! zWqEx@^4d!B`p~I-NV{pjRbC^Hm)9MT0m+MI`JuK=jb5_F>01u#XlNf)L#{9EM`jNV z)~Y1%r^Iayv1bfQ_rabifjv_KdtSEOavwIwVb9yhp114voTQAr4}1G*3GbI5Y=24~ zc1<+SM(%}S!6zg~N6`rWi)aso+Fi zye;&4yMg}PK)<8YPUQ>!h&n(N46|cEiFEat3wM}+J+ta;SfA{j1wUNx`&2m1rgjqpOsPzbq&u6N4Y5toB2i#aa5^$GsEEDvw}vx$u>-y*@d^27m*D}RB* zgrA>;5yL+kh4KD34G!DJqS_Wju1?VjCr3v@k??%*@o|TAC)|MpP1aW@4Me04Epf)sLpp9H)0B}a9R|0Xskz;=^zrJX{uZ+(^0$;Q`*hvK?99~ok;3S6C^AM~HE;gO+!re>NyUUUC8%23BH{1Q-G&m3bM;wnvpoIUjfBo-| zM`*DZ0w5#yAQf7S`2$AdHs)g+G#`6j$oMJv+m`>x-@Z@&c0UcVZ2o}ZGQ{7G{7wFL zL(_5o_60zqzx|Ve_8RCr4D@XVdZte6{#FL`yZvns6ml|uJ8!qU?{`Shf z@VB3FfBSh0{Oy*_n!ok-L_|RLO|~gpFhB<0!coQ${i$a`nRoLt@5#5~2O%QRM!pU! zD%#7(`iiZs)UEj34dez-bt^vZ5EgL$^HC{Ul!e4!(y_tq>BJQHY3AVjS0SJ%=U7(Y zxKkGfo#n9>YxPTX$$K}eW6t}qPs>W|Y~xzEwR%IyH5}fPX}|nhI><2oS7kJEKSEV9J!fcoAHQN+r@ks)OT@5@sL z%ly)D+EWHo1-ua#ex+&ed@^s}kl{$6z3&5P`2~n2YuY>drN-F#CS&Z)8MEI8<2l5r zYHz#F_yK2ZLxK_q5@Qq?nf6}Lxt`-(uXC<@Ao%*;#6{Yx*10^KYZd3JBCgYji?o-e zbKT6jia6JD;u-`lXzxSxHvfhO-ra-qJ6(T&Ba!<0zxc}W`g%do=aA;ALp;pCNbS603kWV%`^;N^;*|Dp`Pp{2%_f zPkl1lt?BMso$(sZn8q2OfeP!dBSuwsLv+SLoH3d+PAA5zN#4}+e*BmA+`X@}=N?0X z=e`fI*I!0lq`R#;S2O3@$+=pHE0(xOcTekFPjar8IoBrQ8U-%s?u<>)UHIG3U5&0U z+9H6yWc>XnV}ILZ5SQ53ScKAe6hxQ2v7hmtd3l5Ey^nj_k5^g-=vrfCVImd0nx|N< z=a-?iPf!h-JOnQ*miuwl8?o+lO@4o?XY$*C1oB&ew>azR+Z3AK?$Q~nIO9W{@ds$7 zK8F}pz1^%c&g6`Robd@_yoF>P^r z{+ZT>01Z*TAxDv*4Sk7g2DqTNtVZZ<;#<(0?l0B8qVHE2>2%NX^bzF|-U%5PbRdd9(i{{@ldBrsc#&xs-gm=qS9CUhXGE9a_hZqOxDlCW z*xQWAycoTPBkQAUdFIXN24oPa#$psDBBomv-9`uqyk%uWHkOeHe(+jTsLcV$rR?x8 z$KXRpD^_f#D$YbB)$@!$c}LSr4ZcY>mzM?O{d;hNjlBZ)AhBZda*%YXG(#(AvSn`) z-oi_zxNN@Cy??%drd8k>im+zk$`945Y9-;fA4rz!I) zoXVdoSq(b%HM_e1#?ONuFTZEl&3J`Qlizx>KPJCV5haJ5PAI>`=2TWYXLD)=9t?AO zf>%Hns~Z1|+uGhwly7i7o-TXGv!i_0YT6QJ@uNdd1&>}3F9!i(yz-Yqy0Z+8!* zm)5b>bcB`-Ljw1JUFo1eKCL6H)r&zpz02$_?=(A(qPu$2xp#I~C)=RncV*CBohgt)&WF82PfB>-T1`92u7ppl)eXovUI|M}*lDd^kGxtdJd)Rl ztYZ@n1fx>0A}Z5Swby-aqIYC;q8IzEc)Oe1Q5SJO!Z@4K@NR8In)=eXQ+X_UCGO*q zAZEboi}lG$1KYVI(NujyJ2~EnxBsd6%D4W(eC7K{;444CW?TJd+>4EXU-F+@hYgs$ z0T=6`5i9lE?&7yS%x`-LBV>IWzwMVdaNBpUk@KrWeIzm!Dex*xl=sK(?F(NQ@GQ1r zBAEtuSQe-#P}dRVxx_j!2+e7UoX1Bp!;%QRG`rggh zXTZ}=%~*Os6i{q=93r8qaeIRgEWa;k2XEjM8SH>rmwJQy1Kkpw2(iJRFgv)PapD!~ zZj3!1^9IlflN%7j&-YIM?f(%THVuaKZAzYh8n+DURLO?dB~yArq;TEoJ};1`AY z#v{Tv-A3-k3D;s>w-I|ccBGAIP-YCh&T}8Vv`OMm@m`Duwf?=QG-<4*&05_I9m7}9 z#v^vhhTOQ0ZHGS<+zxkJ>EOTtX9A8Y1|9)7-)5v&gecZR281VVVebr)yFx+O3nlX@ zq@t_$SgT(~#xoXYU_YQ6u!66|`|0TZRcN{3b1+EY$T0?Lo4@TVQf+h4?{vJZ+0hVu zHGtdE1l8wEP+iUh)#FT%QBLKz5rPMAfRbY!vvPde8loa=ecHJHj8PFytWEZ4cpI9CL*57t0@LgT|^!`$!@Ja zN^{grW^>WGIIrXGZ9K6_s?hMg}mV+bIDn!m%KRX-OB zL=`5#0@WnE#GDuPTW!3$OZR6d(8vF0=0h@$SclAqGGWBJ?ib^W4yUYpI&JvhnGgNr zGtD}v%oACM!Z=UWI=Y(=t$SB9mA^gDOl1QSn9AD}ufbcKRPFEIZ_||MU(1wu1PPRQ7=uOq>s%YRpsV|xe>*{cKZW()nXGoS z-it&#%-4JSKVj|t&(?eE+0DXwZ!?(~n{x)RM{p|ZJ*wSF*L$bX-eIgN)ZX4FY;V}_ zY47HbNmxRAKYC7U?~#3{)?TXJf1|w#tSZ!AIumWK-vNy8KU~j$^iM;3_mI&t{q{b^ z_EPPBLwm8W&(6pk<1q|+e~0bbV`|1W?)7c^1XeX_Kb-_WLHik>q5U+c(I<3x|1xM_ zo*(SN=i^oMh!v0Hg*5ugTC#UpWTK}ld2j=&>OkN&8K-0WZ>7Fk(cNu+19eI_l#CT`LEll%EgnxQ&t>B-u`q;zDbErO^WiespGuW!NyaHYTVHbqvG+nZnvS=kh znJCIo7*0JLM{};5KVWjw-XZ}Wg z@i<}5sp-o_`f??FTt3*9@D=H+?PU5o17~pgOvZK+eeL~#IDe1625r>zHDC?X*9AzR zuWz4%zTW2gx*HzDU-}H`E9h@ct*ui-`6P^apI7#T z?HA^s!Sy6tzN=fiCBdx++WiPFx~8%k(uO-;usdNmGKO)U>LecF$oqoxBdp))Y{2E1 z8tXGeT8MvyBV>3qc8H7%aTY8%=T&XifA%WK%{Sz7B$)N@q4S?BPG`Y_`ya!sf5~cE zUI(9NHXojg=Y{CQ{YOn7k0AViGJViYlsoc zi#W^f^_u9~9$=!|j|8H7`>zn4x+ORLPY_+=!}Psfyl<%2Ro9RhTf0c1VYg_;CSd!DK)WHd56n_=V6miy0hu z>gYS-yN&U|c#FH;eHB+#w4Mm|C-&whePV9xcx*Ucf`vn{gLx;~u`h2&z((%fp&i~q zzy@#rQV?Zupc8eXfDOL-Mxl`aRnc397bce>cGe;H?&L9)jf8{7jW7r>HUMB74`Wl~ zt#^M#RLI$yEkMwr!NLPyEC5D9Ioe>Sw?(27y{VB1uHfLulvcK%ftX;w# zW7&Md9zU9Ko~Xyi=qsX|H&Kk~@L?8Yaqu9^EiY>}IID));Cv*o!3ht-2LFZb+kP32 z)r{uMaXRx=oY}#dgqa9KiK>okImmN&?0R^I2BvKOwd8*Qdpl;)WihN0vYR~(aRL?*Hsh)=pUVn(BN?Y=~Yc;O6Kk)vK1YEoC z2iIXP>m-{=^5-|GVGc{axpgocH3f|2NMgt$vG1I=ENAmh6VDt>2@V{HdNtq7SZ| z^gPmK{9)`ol6r&5d89DLdGhl}iR&~4UFTy8N=E_(6(T-TKLF2QYCK!#I+Jr<%DFO$ zYaqDb+wOY^YFvCjMnL^M@9#U0gxEfvL4wm`=f#wdDaA3Qsr174bQ}Gm7r>{-(802M zFJnIOC+n-p(V1+RXI`khFLPPesxFB4r7-jwOPbP*GdC3Z^^SmVs%l`b4!fsn!%Q;{ z&A~AR4|je-mdh0S3j+3hn89g8WcX=BWE|`?A~O6mBKvzT?Wf1>C7-`gEsxIfyog$` zgF>}Z<^|py-f8tAc5+4A~;e%PE);I z=Q@{jUB$V6B`$RYUxm)=Uw$8({D|`(?R-r%J#wbpBc_+)aO6mU2 zUv=J^FB9&@0-DEEFPL}xc->odiVZ@?f2964r1gmIsDBd{IWZl{griZ ze;-?`htU2;3wd#VtOtK`rF{V38tTPt(7i7>`3)Xv^k>#Hb4&C6T|56u9!qIqOgx$!qGMvXy`2QOFif)Y^>Gb{Rt6K(fjryEu1f!3Bf7*#ZG zkL0)G>x8IBxZ(ElMwb2fu;THdCk&3Tgfb~9SJOL44PX1Aa@_>@ErcJKC&T$ z|Hy;&@biqP5Kr%o>SHDr?7frpP|vm4D4;`mXF&qy5!aU2xIWAdhKu%1^}#y?!Nr8; zLp-dIEiUa3i;Q~k9#yGFA}w&0@3?cwleiwl1$s`^|8P9lGdKLL=0qnZ8aVcx`8 z?*Kd;lgSGh$MtyZqIhs^#21|&WY1$?bmB)q@$IN)Jnt~t$c{;a)TXP6IMzOcj)8Wj zCH!oy{s=6x_j)>k`k5$A{VsC2bk=d5&h#>Ni0fn)wqt56n#Vba;|%ctBWh65IwyGf&L18Oe1}?q~ZH7)vq#;>0^g0VH))D z3?U)Nm6|@z5cM&Q`A|H+64zZhL%BXeecDFoV>hIy`m~Lj`yzeJBz?4|;frq7 zZwK`OpEeWvU_It|eQeit^0KZI`kPexN1fC@!xcmOHvR*(2E}|03)Oo}GGdrsFA1G5bhjz9C&C7_1BN!1_2%G23*m7S8o4=c*yD;aoA-!gKkDRkOkW zc>e|c>=5;ny-d>&KQOP3aWggjwBfod=Nhh`qroxmurpy3^s`UZ50TdDHc-63+(G); znU?UC)`?huIqFKd1Nx~aw&V4)!-e+>tm}^8(oy`8QET;U;Nx9~KL4ew2{~8*_itQR zG(Y#Jt~2Or44jIts|xH0oLpDotc;WCD)~uGSL2s6T}?v*U6rAi)Te2>I#=f!%DJxK zToI^N{ba7I2kwThmU_s$3G+9De|s1DB1cA13_ABFt1p82EcnPy13pXGK(7!%JlxA) z`*(vJCX-8KBXSwG1?WY>i8#-RVBUoEp%5Gm%?MR?^RnN;ziA7l30Nonxsud@V@5`2 zKWdF8juB)`z9GqEGk7 z4IKodZ!EGCVKFSjMKIYY0vq?IV^Q%Ce>#bJ3ll*Jb_PfUw_w8ajT3L9q$$Pm5QhW* z@j4tA3H>?1NXbMhU%qG}m3Vy!WUnlp`)5roLzgnKT!92)`I%xks#qRCH}x-FVbFhw zd}6TbPQ$vI+kh8;Vp3*NM#h+C6mfc z*c~9Ls~@qTdShFFO|m{T)YA!KA6;N@|8QdYBst-L zwdzvJC;VuwnuttF!Uxu>TN$#?T6H%=-p8k2kx5Q?FB$H(pS9{uP)_9_=2@(FIw+P8 zf5hJ9OT%BcC9|W5P`rP`b0py=C;rH>x-@?ex`l}OpG*3YV)?K#yl$9+*Tm?xR3!!P zi6!H;R3*hh8AO9p@QIaC-;*Cve&qMPhWiq6(Uz`e3CQ1%oz_8!TxR)4@ z5hI_5Ma1Zg7jwojobf|q>>|cL@)Elq(OUiMLe}cjIAc9AN(i^G_uM|!xjx}sek5q} zQ^XZUT(tK4htBm9=i11*yu=kwTohFI>0DmU^%&=J6ITy#VOARkcjN!z4(5=%>2HXv z(Pkju%%Wq-!q)+|{Dtfu!HXAEv;P=sj@*e%3c zq)9fU6hT0rVKESNqSSbbZ_pRR>CEj=b+b|5dSrW!m$9m0T9a3C{FbBojF_q+mcdqy`i?Vg%c$=-*YYg!Jd3Pn>_BVvW9(M-ZDnnrrV=3E1w+Fo z+1L?qn&S!kUHW%iZ+n(ROWWM%(OoBauRM`)o@iS}4DDiDW(nVxx&2|y8jH)BHC7^l zomImz*Qaya_+}|=qjk}7`&V2(?9D~|O*k$e_U0l&pZpzg_U0mI+eNbAJRc*Q=(!dG zdKWoac}MmvB1!Kd%5w*a@KfUUPQdn|>ZR8GnSaykA2H zdr1R(<4KX8&K^u8)->$??ZL!A_250xX3PvAp7T+bdBEz+r zTXq!^wEt;%x%#_!%T~e9_?Iu_v%ll|Uj{#u+uhlNnWzGtJxGJUVaJg(N%W}s1+3;~ zM-QTL+wn&aXx559Bg^Kjx31FkkzdO6u?PujJ%J)J>YVk<#i;ex1%f_;^O1D*{4|bB z2;Ugd!n@BrNs<#z6R(H1^%dI3i0GfpJ*o$aGckJM?d?>xz3gKp>})~kQHzo6c_(&z z!SezqkaznU6M0{FLV2&pZ6}uZ(FZhncb0Jbzn#hZ6AH(u@-BkM@y{s#59B=(&kvDz zGHg(ncOF9izg6D6y=L;>{AaaioM4za#q-{aSW(m&qnU`!zDIMt_Q(@{k49fVu+h6) z+l*ud3APyp%hzRfH8v(rDl6EdP}y$m@tG&Ok&nJ;41UjC=ug4pDyMWFY&?Gg#C8$H zcA|kYigBK3pcFTwLeGPl*H0VVDZm!owxg{*Ve9Vk$C(&{TmOH9?XzK@b`3tD{eV;0 zx?{U?Cu`+!_NY^DWnf|#&Qqi}TS$@k(3mY^|Io^Z%eZw(|;^V~a(Cf#hZQxB9EG8^?zKpM4tFk=qzoAI`NC;Tp~T6}#d7 zZYqKM)92%)9~uEjKdZ91ekzgrbB@v91<>{9pBAKv-=D;3Zoe7Rs&`b(z(Uz^sL4{k zAX#}QQTZTI`3RF-@t|6RfA6D^JVx6^^evHOr6LpG5>Ya<(aA2M=Buwtx|G^Vkm6`M zRBzr{ym^-)fe^^v>#Ilzw0JpmuO@_pw=y9_aIOc5t6CFPtIqX4=lTi>m4z6NCaPr2 zz5G`dcR!z)mQIHH6xCSyiP|lqur1|L)>Z!u6aF6+#ImTeS zfy@yB5%oyN1ZIvDDk5{FKuLv|B^Brk4-V)7^PX^_Xnn|;lVN4EYA2Wk# ze14B6girIC5WYtOAv}c>ob?-d^QPR6=3PBs>>rwa^VfhE)0*OY);L`U^|e9 z>FX1`lHBwQV^47e z=D^BA;2zb%0a@r6et>1LFeUw3pe9;eKK_uZ--nof^Wjso$*01#W>eHzk>?^QQRIAR z{IU~iTwH)c125jK>HPUzrt>$DK<8z2wzBWpsy6S^xvDtVL!7Gy-MYRPagjE&b*@`D zS25=rM*#sAJ#$Y(iQUQNAiSc}{PrRB67-@)i_QvX}fVX%LpJXnj^tuDsNTN=i0JZ>1fyc-#- z>shTkU4NG*+9`9HXtR+(v{AH~k*EpnGM#G}=enA6MbdI!722N*A+&q)kM~z}evIZ* zsyp6~N?}aGV1+%8Y)o1*lM&WU4{IQyY?p_%$~}^7X~@x^H{%=m@H+U@VBG6ymhH2Q zmRCD-TKNr8X*nMXHw;WszDQBtWiF22Ytu+F>M!ue(ln!nEi>P61F3 z^}RLX+pY@=iVzUV@B2M>W_C9T)b@G&Kc9a-pX|)sd*;5JbMCq4o^$S<#K{WnG^fvk zHl{zhS}?s27hw9YbP7zfG2P%my`-SFDyZMXq-kS%DCA5u@0w(L)r7=(Pn5uV_%+1m zd119O$K>e@h);-cO++{&;g64fjD~~L)XA4(2*fw9B`Vv2*Mijf?RV69>fXqB9Nq?d zl^!18E<0XqRy<*H9t2pBVHA)XPvgwOiJ5x%I~Cqug>W`7sTO<$+3+IZ`$vt1;UYLI zize{fgTeQBFlma(N)LZYN?eFHx!Bgt0dDvYI@qoLlCWE)tZb}q1;<14$GD#nu+cvq z8b6$wb8rFrYaq;5RTKRK(Q177r%1sMbKrlb;4f70y>3kqVg*D*^uB5B#=nY}bj~9^ z9`x{rw|J?5ej{-!MblT^(3gC<3+I2{eV*gPZvcNOO=m0m}p3ULLGN zE_5X$d<)`To~V@f|H$dt2){u5>)|Kv=fz|!OkkO$yp5^5Q;J-yF#KkfO_4XC*BhXTfzQ2b zA=;TR$$G<^z2UdS`-Q_BWteO{!XY1LJ12cu zr`zH4@;IRpHFw8*zWgyz%~+z5PxEh)lvT_x?Ociset8tfbZDUka`P1q?~t2udH7<1 z+}+v_dn$0J=kqYV19Eo@(t5!nEa6BI*C;h(V66(lhMgy=GxgUIYc%qEtoUTqYhj%m zoCdM&&a!7&B_54%og(u}Mdtg3%*m5T(jg|nTt(SK3v9|ByizE8>{UY9A7R#5wH}m> z(XEL=`MLwRO#$w}1uSqn0k;!yu>$3byEmvBV1Gqo|E>VE-QUJa| zK+Baj$pO4v0bZj3w-WI01hiaf=R1JsDZmR9;IjmLlYmj8&CKZL0J;=lF9rA@0pB8^ zcLQl?Zxv;(}fW2>3PuEmztz4&YM?aJ>S&j)3m~P;}i|2Wqi`dPqSP6Y5<; zQP-W}KuuRra}?ACgnExq)Q(0uPz4I=Vg+>?m-p60{~U&6^brg!?rukHoy>y22g^{i zu&fRkdxObD9ZqM0B5Z4Fk5$-Q^QS9>Yo5aeTob{(w5kgNeuNpS9H<2fYKej>BUD#H zkr{sJKuuOq*D0toY5}DyV^seLUapbD;zE3k5YnL7jSIqR-F%8hu`NWm5bKCUndbje{G50%K2=-Ag7Z*wU(Zgdi&--3v>6qC?y(p;<|z2I&D`HpH5^3fy&6A(Ms z(FA10;{zcn-y=?}kDo)B8S*;U$706d20}I4j=}PIG$uTCmMqqy!$)J%JIETT zUmcPy)Hp%*>h>qdeirQ_y+iDZeX9EL=$+#Q81gXsE~aTluJBNG2Cnc-b;hlXk+2mX zftu_4wqry`*8)C8pHd&TjTOP#D*K6?_^qSxf2afh{|tRP3jgDU zf5rcE><<6#GR6P6kpE#hwL6f*4jh{NZ}2~Y(o*?%POD9W={2Vornc6R#tT(dphQcG4O#hOR7K1U0Zng0` zx>)d9j0^C325hf3UROb+L?4}`v&ck;uzetVMG8JA8NB zKh27jL%H*O^}_XV6og@gjg?}%ooB< zAp9=f?2GmCQR6&Pgvon$$jk+Pn+-->B5dHp1#Ivmu9PkL@DxNt^r6e!%Lm&Zn~YuZ z#6-jsI!KL{Rkg!ti0mdj@nfiIAQ6ZJ}N(K;Ck>anZ~~h z9BA`rhIVPShfrt{zSZGx??OTNV)QSS>_^hh*tM4?`8z*=dqGJJus3eN8;DfI4Vc5@ zF2D`A12K_`y}9AlhlQHj>za>h<5K}rPTew5x3Y)~MZGe+1ObKyaCU*2)dVXk%IrzT)H#)?k^34LlKj)cU@}UlUuWZK;bU-ScKig)9^Ck#8T!0JN0d~_>9z;=! zuGqtY>aL*rD5y`lK(m-;9>hj;MlodvTW+5xQGBTn(JmJ99n3TCC(&^g{u~`HEl!qUX=IMv<-3{NiHB zC441_&i1Q+!*Rq$iue9eZM5!f| zpJa<_(cre&O>LxkM&u@&G^S5T^B^uD&22E9u9|Oi=rs=1GzE35g7Of`;?Val0|7U{ zKBe~8Ce7Er@Mc9XndtT>h*e9xBm}oLaYXN`u&86t8jp}`gf<&sum4^$4z%%w%N(+_ zlXZyLOJF8hk^>{R-)LjxHbXFS2QI+KHCzP}BcR)t7>Ln#C!nv+`rzODJ_-zIkn>*J z(Dr-7-ys;?;@^uD|4steaPbH1$Y*q+B(|FgC`4x(!?~a6h$q^n~H4J7$t8$-xGMYbG}plpPOeX;P>q z!zc=$OP+kuEHd`yxfWwD0|c5#y~|?k2}xBH1^4;;DoGW7N1X>GuiuH+F&sq78n6^% zi;o*-0&W<5p%Nt;VatH}w6d(xT$;+@`(m*1lfhTaw8{DBi-epn-~w{q4}Rts^N+^rL4_*SIe=^R}-;?-T5fibhN+n7V^S5YfC;a_41h1uNMGa&$;qOmj zu>ZdNP1;de5+*0X+A(>D3Efultl(xK=g#YG@r8IR3A*adMlk#y>8=$s!1$ReP=RkD(uZ*GL}!VjoCgqLTfz z-AHnjPuVOGHqSeJOkUY^-1WmOKYaB%yi&3RNE(k1l^U)|sC#-7vI{(%en_sK6&c<2IY`6w2C9t%(QZ!q44c9QV7tCv@INgKQ`dJu7Ln3wtWz+a(0 zvumh+evdYsN#I|PsoeR3;8ipuUGCIdtm>Vn>b)VU-j^XNqR*nxcIPSM!;d5t#f5Tf zwMh$%@Tbwo(8@y7<%YAbHl7QZ61NKrmMaFo7eAS;Px8;`ZR>S-zw=+2H^gW9-_I1a z!yil1F)UF$8d0*?5rIt(!S^#ODUe+5cmr!VfMd(3sFcJ1RlvQZx&c)h%MD*YJ{8cA z+qQwx6fPtDs2lI5#SJeU&gFXOJ^y({;qOG_z@k|Xe-VwphPJ~U!ZkGaC;bzq=;&|m zWaT{DqiwS{a2NsTsvK=$+wo{hWXxfd*N9C+*rJb}8t<5+C$w!0w2qv!#W4TK3JvoK z;p<4x6Mg{#+aIW=`v`CB^FM|vx5z0ru`_Y_J$FhWz21MP7)U+l^~HucuRIa5=Nx!5 zz;De^hk*smP;gGe%rpsAGG_ctT(Ip;3%;0TtJ2m4MIFmB6ELjbgFSby*rR~55u*jO zG4rG%2fMYR@4*rndk^Lxr|rF+w(H@~qu1Bks4^i8Li@DfMgZfeu5t$~wltIVSY1zj zeSrPC&xG|0-!Th@<@H}uus{h7J=mLO^Y5H}qxkZ{X!YfTe$f3vajdXp_m1I)KTq{i zISbR#PvE2{aAHdx%TrJH7m!gH{u109`x?i39-(!$aGEJTFcBk3Uv~UEI3`OkEjCJj zImsx!y~HRjodW+vxo%F#)=eFonG4Vi9c?S2?(sokF(mAzg_lITHUlegqR}$K!qDz) zMRwfb95ZX#CY$h}=vtcgNCdob^G0N;y#a5&W23qmo&~7yh*`qoufp92n_Ml1QCr)r zA`afnKHxB&c~C?CR0(LWc^T^efZYGuYVra`iV!ROLF~hBM?^-|Vh0%(rT>XIm{dwL z;>?HgK*~1%0O49A`~~=A@TTa+h!xVFL!_RSj9t*{q5~3z-T(~mlE&bpbymqX8bK$N zprM4=W$+iYs{%dYwyaIjaaa*{sA7|8wcBDbtBTW6g<;+f%0YO*mV%+L3hBXRC0XOJ zK%$i>c_}n_g(LC|TmBWoMWDu%$KL)HH`V~wf-mEVJ6bW?P$Y;f;L?8-!IVBIrzodz zQPJ^*8;Ww^Ml5j`fUCQ*Av(V4QG_IVn2NHY8PWCw_Vrksrm+(!4+SG?1q!!Bt^5e# zv>-3bJRW7{=w=b#9i5F5i}11CoUV+lyv)Lo{_}w=OhiOXyso!;r937OtM5) z@__m&+Nk7|l1tIAH;~Fu8-$sMqA11QAAk>HEr<~~(ipq$$xdpW)_JXKTUD*LU{>_d zv0A{?rJDsx44OY}&){tidylw*-Dwxse29fBdp7^j4wU6Olwa`6U zT3UV)rlqHGLH-M4p*DnH7G8pHEavgwu#TuJyfpe% zJ+OcYf#Thf9(C*>+JQNRhCm+VXH6CpQe?Z0P%_v_GZi}JdkW|n3K%^B!BdzMuqENd z)OD)h=(Fqj1}m`QdcYcfR26<#jtVf)92`=GqdDZ`spWD$44O*MgbYPBauCW#Ie4A8 zu!wMbj%?n?;!%d(_C5(k#xnPyBsTpjGz8SuBllW{_GPpr%98RPcoBr=N%Py0zCe7U z^k=;ndMEV774AwPfs-6+0e?rMPve74#pHsQg$v$dZy_YtMgI!VlgRK~G}+bxv|u-O z0H!pvw;t(vOg@@4+_g<8iUUu2P`i^)xsJ)k*6`YFilU_e?TpPrR-z>pc*2QA#fYL9h7-pMaGI1(UNn32OTyWxN17G8 z02-DL^9SbLU0QGq-ZDo|GRKw#TDk^4>}p;*CBSc2O~m&0{+LRdJwZ8xqZxbe$uP~> zcaNgdfTTQ-9Us8*xchlQ*yvoq}To1n!UET;l z9!rqm>t_HcERo=ZWEpumqUtb-K}K>HP*NnC$jD)m%X3SftVFi;wN^()l9=`+5i3MR z2}F#<6R>BFMZoFk5|23+<0{didZ_;KF$rj`BXnRVw$)^392}r_rhJMkO91nHfc{>W z7G3iSI)jfVHvj_@{nNv-=xtwG6AG9}#7pc}zzwdKHHv^I z$tvKPs=GCU-;M4R#}cFu7H_d}EI)F}q@g(O&+PlU&+H2e-d5|oT59|TFLiPCxRt{b z(HraJlw_vGN+9MF$G2_AtP2sqqmRb)kn>hj&?zX?FmAoDVj4%kL72|gaS zRD@t=my!s{M+xe3l;9(PV?Dd^hzUg%Q-HU82nO7!g{qi>GEh&Eu7zgfQ8Cx&v8j8K zWy2IaDM>6-&F9f6n1o|RNKMdd1YzunQq~s9^!fmCtPZ^>0I<_%LXzdr@-Ws3n4_am z<4@9$!9{Zrxue<$<@k%(>fLOnt(C39Q)&~EM}CQF0!Drg>W}?H_J;OLHIbf(5S6w5rR48H&1^16`DS0bo$T7u=HOa zIDjF^I6ury7?Gei&t|li9~h9n@V4lR=OB|{a#H|4vk3i%T|>IR9yPt=3x9wLTJ6^g zFDabJL6ci}>4fORsy})0^OFlmdf9#p2*8V%Mz4B41p$&xt;Ql|v{oEWSgp#0Y5q>=h5gnIh^M@$8te-diWunR)OS}k540EbG(?4DC zLT3{=3pxW1KcJure`cZNfoFk|>+L+uf>5x=c7hTq4X>r3WU=aDUL2aUDx3u*p+xFY z*q8v^Q>iNn^u`B@Cp9?O; zm;QJ_eJjL+#4I{5sd_+R^-`L=ghXi^6A_=xm$WBe2O3_%m3;LINDl4z^9dLhQ97G} zhuDFg=SOH$G0LQ_+yXs~MtFx>%c8zc?SR)FJ&0q_s3da43^`x3i{Zz{mga6u0rX4KZB zaD~K%E5Iimz`rQK7Zu)tgy$y#gHK0RBV)j#hx*A}Dy(lkD|<3h-D5u!jOXSpjY*;L`-WQ339_*zR@9 z`GTH3xS-dM6Yy;U7Ae404&d_&aH9gOB;a#!9>n@8z$Fe~tpa>l0Zt^~Y65;OI#R|p z4&XEec&h>&NWjxcnU@vd1rFc{1?W?N83gQ4z(*8dF9+}h1?X0Q@6&+3hIlMjfIBDH zz5aZl^!i&|(CeoOSWm$53UIvx*q{KnE5JJlm`3!RtpI=P0N$?vA6J0CCSW>yO+5~@ zzS#l1NdZ$d?ouL5N5-^8=lNI1=4&W;a@O@l>o}~orMZiG{@KFcw4+`*U1$eRtfPD$r zT>;+i0N$nm?@)j_qXBp-@%T^ZqG;nK4&Vd@_-h5&m4K%a(Aufc-vRuw0vx6QzZ(U> zJOWxf71A8Q!(1OkdT9!<8F9q&C0%=`!dv5P^!!}`ay@}wAJ3|1u|#X9!e1T06$)^z z0zAm7ZziC%Qz7I41{B~@1^590XA>|g^J+$k19*i3yj}q|67UuRT00ef>HrQ_fTI-P zK?EGH%HU9-1_L}E;{awVz>^f zD5%#IRESV}3B};_`yHs?D5%F2)UOED3Mec=vWCEWwf}sqC{C+HJJTlU=A1lwT>P-m zWa?de>!zkhs<(n+p>5B(NjJx1)6=LtJ?zSX6)GolC3A3-&9`|Yk?Nj^E3}oWEN<>n1x89{!t&<8OBd;qY zI)*-l3z{7|4^yt0@tk2}>zQ)|TTQqCTMq(i)mmuC06piY=wUDEp$AI?#@`_x6TVCx zep>NS{FyE1X0EQ#rcnJ;s<)f9nmv~=De5HqvW3LVKn#}Q$AtNkbAa<5TuN{b081^BC{~6tUC@pQY=^CM2lHUB14Vh)r z4Q$`AqM8}CY2nCeZ)5r~<*$9k`(LR}K(7)sD6Cw79?E=^q!FYEh*fz4)M-{O}#;%a?DpGm|Ea6hZ^yoR=~#YB{*jsIK^G-t2w zQvkZ`i_S(Nh30XgOL_=-qx(=AJBp$P%11chcnFy6WU%592}|D zEQ0Lwl^XY^`O;%zGA(TT&@3p)kA^?jjID?+(>#BVzhFW;|Kl54^~0zx{s-Nxti+9- zcn1?BhPjA|y19g^RiT-#M_iRgr03dSbWbZXjfOl={xMpxl$7C)x=63RXLU;(t3By@ z89L-BD(o0S_~{4fciY+B(#Dl$-H~h&yA=%;i&*S*DPsw1Gmn>c1+pZmk~PC7%%w!L z^??onvkp3li=0tQ4BQyz)!82N7{v8zLE0S<(1M(404iLC$F$gk z__aD}n9Ed0bD$3g$08$e5WpIN8Au6j9JfC6;y!fjVfk_~n{_c4INQ!x%YS@wIU4@F$(3mtM9mgjeJk`tPFaH)8rxRCLF9wEnp_vzp{Lx!u`*>qY8G&Kx zT1}dGKMROak2^E=HS}!|r7{79kH5e+tM~CXe3<85jdZEam%h#0a3Iec`0$XCk7d^Q zm(k`|?3xU1G~xj^d$lopV#`sYSx_9PKY+v2v{8SLJ-`GA3gHUkAKRos`4hmDUNcu6 zw2ft3e7pmM+;7fI$(@iLA0+%28;jB`46Z}lxEALMkt=%s<^f4v6tC;~_?pd_uVN?i zja9&QT3{IP{w!`h_`pqJVBKkjfgLSH+LLeiO1EQ)a!H;iyrbds(>&=r0y_@t*jm;V zK35XMNZ;PL<8S-dIm1TP5PK0WJ~8RUo}fPwvw#+kHnemL)N|)~`Ub4>_JyyTb*aKiU1^6NX|3*M- z?%eDEzN!HKfeT8!mw=ZQVnohDy~`b_|5i}XE2wUmXjeUi;$-fucAzR1)Ljbd04B#( zzb6#u&JqXe3I%n&g4#)_hXDoQx)s7T`a(ZOoH+4;m+9e8WR9`6^iL8O|HB+^rDL*v zjK^4q2OT@W{MZgYKNF^b50PMYhs-ZQFn;g$@gMj&#zy}roW{E)GcXqpid}(<+T64S zpQ4e^7uooG7W@VLZNLR+S~2X=M*jR zgo~Ep|Iy~6S?Q`=hM@fY~iDax;z<@1j*df&$Hv8}ML6Zu*6{VOQ6}z$dNIDs+<0 z5vmg~l}2-ZjDgzgVSsUE4;vr~T9Xby^ere{(+3r|!_QcA#Mzh(GtO2ej6n(BKqD-uAC-NKmp8UmF2Mxr z!O`Q9X>TVj;R8Dk4c^49&S~+JQsRXI4K5v@Y6zYJI;u9K6JO3gkv{|ZV?<-@Vz$ui`PC@U(d-{?j?Qu;Au{yTVPIkg1FQr4 z&g>5>%5o9%85;qqo5KLIAy3%-EnL9vPxk}6FDVpu9|8F2kI!Ps8TUKzzftgyDfj^4 z|ES=z2)~~dWB;!MKV8AkQSdp0zgEG&55h;A75p#<{$~pQLIwZXj{$$3f?q-S`xJbp z1K&--pQzwpCHxHvzJl=A0N&ECQuL`|2xEiRipeKaH)mubbcX8EAam$*zwe$FINU`( z6rfd;zhO0G;OL-;XAngC{(aa!!2P)UxaYP(Z}{+7BjT=tE=4(Kj+~44jmQFwr7tm- zK5LB*1qONWlNw70qfKYFwT(7|XQky~tYY8{8yi`CRvP#@&9WTjGKv=dcm}Dmo9{r40BSC2*+UfH6y-YG&b! z;0`z^X1cRsE=!EhB8VmC>t=|0B-?@|%Gx~Kw{4Z2Ac{Ky#W5&kcbo3kLiYn7^q6NO z6}~$}{xHEr+z7Cjha0R%;J_;k9LCxB5=#PQuXHo)qc3Ojr0)sfxMvurH9Cvk7&|QbSrUTy=;4tmE3)G|P76R>t57r|5V2-bJ zUwjCw^O#fm73J^sf9)B5y0)078^y-qN8$-#E>!(3P50(+&?2v*;Bn^EJ(*+7if42A zcGH0`>zfWPD1c83K}|gC8}04c_#oShgZxim$MF(}6x(ZlMU|}$RI5TFGf|bDc_TBe z64s@ZK-5_!cmc$UzekU48fif+GeKC3P9RYb3=z(=zpm}M*Q&ENo?ClFTKuC7= zfxUZD>TRJs8xPJlA}${41o6GmHO6$MCjqWb0gOKZsds23rCvUy-rUoGnaI$7DwIb_ zy<0IzVmqOOsqf2Ry-NK+tG$oy4u6aDJ&ef6M^2*jJL|tr?v@t+soh?z4l&4KRL|e0 zYZq=q^EExNV>hOlE_s*^yON(uH+u6o&l`7dVHB5eaSY%d*5X^72!Rb~@Ta@I=5czY;wNzGZP99$qGkB>EDR4krYGw}xNuL* z#E3r8HaeWxhx{2ClFt=**)To*EpO!VJ}Im33%ud^dETz$z#V_^|m$n@^_Z~qsTm) zubhQfrY{+9R(yfbT?hCO)`tK-)&g!pC?vj9;_&tQ4)cepVL3AWM0ToI8!n+u^Jf$d zeh(YH<7Z$za?uG-E55IFVY42n2Q8rcr8j!p)*Jcn_zN#Gi{FFnC6ml0leBiqB->Km zxn+MA|80D^>?Y?nkMLCP_se+lzc>g6y>twE&Uoz8z@iumaWy-B$=NZyf7Q$zuDJ$? zy&65mg!O^S{77X&0h2if`J<3;j&*@#Tbw*3pUdQ9k@`|{YA>XsroXAep0%?+#0v+} z2&Y964o(C$V7PTVoj5z?AYmiD*f>3M-pBHE-aB$RYqKN{yaoso|EFx{M`>t>yd z`3&K_z)l#&JnjPdk_Ce#20`10k5Wz#N0Sl1p+5KUj^Lzmf?ZXhd_tNQdLauf3Di}x zV_N8S^|VMm#qs2AYedkP%bVY*EuIM3@1Xb`4kSTwuPe}DP$g3?y>z+s{N-sxc*bP8 z^t_B09LQ7;QuuLScqwkC%Z(s!B5#Ssi|m_`_RS#s=1lvhw|&#YzBzO%UytsSHZ@KO z3CN1LPBG2^1Gf4;?gvi;1?;suOkbf4g^gIMw|Y6GSpHm;DgE`;k-KxvAAvE|=<{fQ|YNg7JTkIUnz9 z<8#%IR0m@%P+*+#G5TbrMusi~m>pIU{oo{|L9Ypp%Y=XkZ@}5c6!wmiH`)c>%rm>< zw=Wcj7imS76+6zlkKRqPMQ~PgYVz$?a$aikR4ch}YO>Et9`qtwkSP0SNT#^eYWqR4 zkZJ1^gLTC(YPWtrg1+T7*KS0CvOs5JKLHhzBw%D4)XZ__3K7+XSc$NoxK3bCam~cl zooFe$na5&&k781j`O%YTO^0OcqTCam#Y!c-ohQ&14dO`8PIOdalJ-O|;>$_BhyJi9 zI#j*dM-R)ZSa^2!s@B=72SI++21QL?#eCA)t8W0RL#;2VSNmD7ifHQC;0HQ;HON;Z zYbn_T>8lHH$DUj-8S{!&YMzyPnN<^_ibU6iYK!dy_@VK&2&BN8A~N(B$6-E#()%Xv zU*~=4kl!S9Ul`QCqrDW1>vge1xRu#rSOyseOWaT5FbbxDr7_th6OGHTJ<&HX9K3K; zly>D8Ya?ComTV>;XU_Zt= zC@cHI{3*g$_i43fK|_G_818vJ(|?j=zFD>>rXe-io>#RlEgOMPxyjOiALxes>qFs* z=w;KXITxO=;VaflePelgYkmXjJ3%yL_-$&Rf-oCgPYLGOtM%bC*Gv5XUWfYc1W0mz zhE$KOgy$=JVF!L@Q@UGC36+7>J3wG4Sfb{G6V+Gdu2YEWHMo9>F?>zO_Hr@D_IBym zUSoJgN_#(mAnmlhJ&({{t0~3Ol#l&4sivF^w8(nQ82)I-_JAhRp4q6Z9sZ2rQ&ZZ* zbF2T8_B8g!>V|R53?0O``}Xq&g~0+W-q9*DkOla_Yu% zkqd;@9G5$KDx68!|3R^*1&>2bI(8Bl@OxVYu<5=CNu_is2Dd*ojIcP=ux=xgW%@{?`MSrd_2Kz6BLl|&c)d6aOTIf4u7Y63JV5{tnx{Ay)%#6OQ zQ4!Z*y4&Gb>E`W<-Fq<6_Va{iBeH+7n?gB^6EE9$D82Iz`QW(`^P?V4mrm8APH&*8 zkB>WZXL7vn;;|T0cpP1kInTudfiaC``OLChsRa*xp+(?KU!2>{6dfr8yylfd_<*H% z(Fp9L^zdO4+kj=*Ff8z92-1kBJAO{GG~k%a35^XE*+m9_eKWBltE4uA>!RJbGF3uL zjEJ3HQ0%R0T7f(xyqvhh=|(8s!)6phW1C#Shki!**HEMuM>#9#W??^ghFhT26w5>dEy+kNPdVR`!6_Q|m)}X8#6rDXhUtHh6hi7$6;eMAHSk8O33mX%b zBG9{MUVjeMd_C-PgF_eOpsYx*o!Q;e!ZEetx)1cmTG^Zbg%qDcQtVOD0^|-KO2DYco zpi(@gtOu<3{bhbmYOg`^crEgH+mWXdh4>=#3~%J2#kh@(`2^t`9OX$^5<8!5owB&CjbfoY zf5zw8k|H5Fvb*BGpqK&*ykC~9Jecv^Wc@+*3Bct+X*2@juo3mbQ3j#3 z!4^uol0*G_iX57nm5@W)u_A}`{1qZ{w9pbLH^gv1I0ZO39Xq1Je_kkb8Rp3O;$10A~Q8s{^1ZfY$+_3aJDPC+dEg;6bX=0Dt35 zTj|-a>U>m{R}5IogPyT61tyLy?HLa_Nek6$H>lT6;cHmL!SxnI|LpVSBv_?tXDdV) z+)GUdu<`M`oS=Ivp5>7O||dXXJ#Gpn7?0%mm5EP{hWoQcw@t*=e*bSZT49&?9E7P`u3_u z{J+B7txfT(-oXER(6&6`eJfpZO2v0wK&3!qwz$=>l;1F~=pxhqH5h(jJxYTZ>msW! zUGr@;=TFenvkq0x_u~-8&$ZCaq%hAl^0JEQ~5t_6od>DLJf_KHvK+}731wQ+^Pmi4P zA+*tW4$ph^1~z1Rs}Ao5sr+YOWV(hx3-W(H)*Ny+J6(_7XZ-E!3~oBx{WY)?r#HSKZa-?OuTc>WzZLyF1KprE{r3t33{DN4QDWfInchA7ta} zMXpW6hKS;Cf=l1blbFXI>a*do& z0@a3`-ZR*T<2t_u+U|DEkMJ6}}DRB-p{C2HGqLJzp}xCNOGZB`3cO z$>jtWXmx3Ko&mZfTMtK`jaS*bq2M)=qO&)Je}GT;qvKidI~!Qa)iw%(UG9j!8+$iz z!K&jk5B9*%8*a16yJ)gRbo~~HjqCP!gn+!|3=6pag0{A%gICn6UM#~4kU7^A{G^x8 zw8(NOJ$1c&q((;cHX}c#)t0b_Ah<1BK49-AT8!37?_yXGDmkLnZlzH+au%mfa6(Z5 z86_6tKuC<2!SxWRp?$hm^d>$d?bST5Lbl=QZC^O%<5MI<`w}|)-6dtD zTr>~H0L=Sbn`G1Nnwc0GcOmw`8-5!@V;UrHl7#vRT816NUuNoo7b?*pNT2udD-uza z7U?yu8``@G@Ad!&b@Lv=iGYM%W|I@&0x7a3)0;o397EocXHaVg@3ZLx-rAq3=U-gz zKUqiUNdE4!UGd|R`cGhlQf@+*w^;~4kSpe~^$7$VCy^9Gd-9?X&`nZC%H?PXSb`82 z^xzs09X+L{iJtDjZ%R*Z$7b#@%ZcYv>H@Cg%{P|-jXs> zE=TL{vOdz^Icd^g>f4?5*L0!3f7ag%&}#H|BrmGJKa-S^ayeRm>#>U& zowyGxW%PF$I?-8wpXh@Ax{wD@gQzj@A?W{Ef5)NK=&z3#)!#9aGEy!_>+jZH(%;9v zlm2o|i2e@2A}eM5zJOKVDo8Z;w}hxM?;&WRi9;f^+TX%3oGgz#lt++P$A~=IQ??5d zh=&oY^$d<9XhZ(xnh~(hD)A zuh$}b)s+4Lr*tpOQk7^_nt!ykosp?|2r$G7?+_NSB50SPn_hFNz!~8ImU{DJ-~h*V z_`)q-+P|W|Jp?F}CeqA5rhRFDMQOG$8*ddz&_sc6(wd!Apw}u;#6`yfuVjIiv^Y#q zw2+{Q0z>TzZ2eZXP{w1&0<&14H5~t96-dxTfkWCA_?T7TGC-wuEbxW442R!KtO5y| zDDWd#xs$tmxmBRWQ5Qn_XphI~EYxDE?pC1$PZV0)uF!N8Dt=t749>^f3AKSb!TL5hgvF#vld@p&uQBSOk7*^Ob*IvdS`xRg8)fX4|78ZPm zR_1zy^{WmZZi~*`cbEs5CipSjit{x6UYMiBP}$VlFJs0YEKOT;aDZj@RP<>txy+%IF%XUK=)r>|vnDj3mRPy#?MrPo_&9{2lDjSs2Rei`5Fc8X+@DzfPi zEdTGaB!pNS?p7F#hUlwcAWvhpctW|JT->rz>s1K_y1r^>`Cq1B=n!HOt4!`sA|5iZ z6DO*xj|m_u^a81@0Kc$I;a80!)IBB>KjZr0=W?BcuAq6$43v$+FQ2txOnJIFCX+Gf z!E)3H%`~(Qo&s#ubW702hoD&E>_1u(`ZAa<9H>MIgu*x9Qb3}IPLq5!;yTHHU*)R- z)Jgstm9NG_C;1Pm{AEX~ze43JPVMCV5|zIa?=wyWD^x_rU31%aQU8l^;D){s5J~8~I1YmkV$TU#$T3l};OBgxwv{`VYnng#)x~ zb(+6c<@57P>@IB)(B~~4qY$Re_DHorqLRUIQiod-_=3tJ2PU>vOC@L)cP1JH>!V1}~P?iMvdMP&JwvavL!=tJFNhr8gJNd-{J^Z@$2|EPF{dpNE8~!li z(x5sJ?j@=Rzl34sqsCN_=>YMWkGeOaS%xXW9*C;5;(>6wb@Z&yK~wIT{b%$J3?%pz z)Da?jK91cl`z5wJSqF}M7CndLw?gu;2XlukHlv=0+wAWXpp-A7Ek_%$61;N zA`9v{0K%{6;cxWt8h0I^B4aYmaX1JsUB`*QG7RC9UnFo?D&VRGo6#EBq+v4%jr4IP z)Q=R^305NxK-YhDX4Go zCV1iCO_S}{N$WK+qH4j>Y?@j8N@nF;9Oo7*tZJi#(jpU4h_@7*X_oGtg*CJ}yN~wd zY;2`$#w8E`{R-3f#aDmtwwpnB1-k&Gtwj0!s6zUjK{$dn=$^$swhqj=`mKz#!nItl zYk{*2q#ev&?4GGIH)?_7k%Ur8tWr3MsZO4JN>V3>IQEr)(RO7-hLN^!XZ^3J2fL(a z*zD4_L)azzBVm_eG;4MngZ-KnXcJHi1j9~35~cYwyrpl(PsER&f7z_E<2*R<(KCFh zo8kF>1Y6C}&3UA_q9|@`8e<`8Q9X6|gU31YZ@^Z>@Rc(>u!*oJLEH$U!VFcSTHTB+ zlFPkHfqYi!UTc>j(J(_qmSILXKNf|yd%%&*_k@?q>GSh(?#R6ujPNDi&4J6$9Bns{ zNp(c1ptH)>{)oGfx+hzWJ_ru26qw0Dw@0$zjPD2fO8X4;R?uB3S?KcwJ7o!Oa54#N zqtKkCn^o#QD568Rb^w7}-bQI~Qro%pe8c{rqjLOv?Q(+dR^)mUMJ>l8Pb5GwoJ2er zql0iQ3boCzfHCg9_=(Bm0U^l>&i^aJT+v9bN1zrDyPU^^V&?I%%X$6q{e$_hBRWwp zg}DUbD%_q97FYrk8@%l#)gRo5cBz`1<9Y_pc3=MfvX4FKU+WM{<_Zx6@uj+{L<9NP zW3CZmU=ueA0|@(|)jo({4%(mD{H)ZIP817Px#i=-DO+ZL{cmrXg^B3kKfvVHCW6|3 z;=bJ`g1K#K6G7}&+!*FFBF(VdwH(CG_|xm0@#aMTbTXWwOWkgvuO$)zNTD|G4H5S%-fc@OeoHkYNA^Lo7!w9fZ&I5grM&v^4IR*JmrF z?oxLFNKdw$p_gv;mA-}k^=jIV{(W9jk5K@wLlK0o2XxTgf~TVJw@he|ggm?s*am@Z zK|KlBm4HPSh6EZU`MXo{dFGFdonitWMCsXbgRwKfPRYd0MUlP_p@g)t5zj*i(s3>-7@ZK=IXW+1s{+rw9`P~aV;dcW&Fi@AuE`I|9u@VJqcQ<(9)4zLN-ph-M-Sh1; z_QytyhXkiVH|2=nya1(pv7hnodN0mezPll>VG*R2RJl|RIq>4hA0H!haX!UFmwrxrGo&N&brkEg--?Q2^T(1 zQBPB_#L6@Bch7Es;JhD27v_IK;kpPYqF^OOAS@oWg~bX>Sga9Y!4b37LS~(6*BTkk zy6qdmmHeVb?nX9ozlc}N^&CfevvGE@O2YmryZy#wgDp2E7qiL6iVOx;R?YI=9Nlo2g6QKc>a|{p+}JFTG+U*Xt|9< zVA-=Uk0}XMN?L(RL+gm%0!f>q(x$XS>z{$ol!*z*#3;v*&>|90+#V%bb&wdck`3T5 zjrc=62&!={#E-Es55L6=S#QZgsTVCqD~TIih#OqKAKLzm!;%shJ1cp;1}RE>cpoP| zl92XUXcuaJ#!Vru)&3f-Lx2-yTADu#!MnHE{D_U)Xgwu_+HZrGT;1?vbd&aXllFI; z(sXEYebe5_o0@h`-rn@_Joj~;`+~3G++3&0 zMk})Ec(vUpp>N4}sUp9*@QX2Bl_$){3!e(q6E1w3qMj1tT?V?Ev9w4s&m~6Zl{Op7 z_^|84_*7LCvkHt)RW=(@kt10cpQ?6baeU(NIgC%9ZD|k96J@+#GMO)dwW^ZFqwv;B z43PO#Q};uWe`*@n`If7iTvbT$)PUuE%M)FfH1I#uYy?Q!6qVK<4IG~sk}KRTBBlN& zT+mKRY1LIp$&UP%_Ta~;0i$LY*ML#8OKPMZv=A*IE^r|(aQS{{`g01JI*dp3|Ki8n zAr0eEtE~mXFbt_8UWFL=sTPrTr%k9y(-Pkq&sJ;2Hq zA*ORdnm-3Pt4NU-3V+MlOH9Us$CP~H&XvwzBb_+QE>0)TvP;sZG(m=?;|Ce|MLK?w zOc}%U4(8BI6TbNY>diMMy z!wV*=TzJ>BP$9Pp0t^odw^fi{cu=UVg6P77!fX}f79JF0t01)SpzvA+*@XuStqo;& zHLX5p>MQ&m+n?W)pPqvTcONM1z_@|46|SN4*gJhh&P@-CWeus?HiaIqxb=N6jgM0>uc zQ1nofg`%&nswZYPiWd5yaKQXgz>qz^V$#eFG#$#p&)&(!0pT?K@}B(0U*1oVdj~6< zMDC}~cTJ*BiU16Eik+zm>{v4f^%1Fm@8lAxbgEQ0Q>v3@G5JsqlMm$}*`5bBr3g4g zq8{k zy|JfroA!;5V5)fW<{Q-`!X({Ey^YVf;eZPEzznK z>od}{YVJ2nw2Zl)r(JAxgxP^^%tmEDLS?Eq`N=*1vE(C6Iw?Q7*WD!_+ijBalY9JG1ABu_yh-`Vy=|8K*S=Hvu~%_J ze(sdPlE=%dlb}z(I*I!9tCNbMPro{;q#K7j)$ANmvvZh?xu+Y(ALpLtvQ1n=wg9R4 zhKe_TtHaLzti(}MPj^m)JH3_J)%TpD@i(S78~`fT8ASA)nauWUA8td}{w>k`#HJZGb7Mw`!_nVNzGELzbF1nz?>xb=D8U zwQQLG)C4~^Hq4gtA>@PB46>6m$YX%BI5{W=Ek919h(*vMQIMSZr!#(bfxOV~Txc)! zI~Upu{mwN7mlEMye~GaA%o1VssU=`_^;!;Iv-Q6}wb%b#?OA$Yf~B4I zkspbqxh3FaGLY&f7O@UL6VF;TPl}WEl?9(|1ksJ8*MKRM!L%HYWMzATWL+dG4wfe* zl6R4)SXiErNajVN;$eA0BDoidiizb3iDX|SDlV2MB$9uTsMt7Y>x){*Cv4m{znmu& zw!yx}@1YNW(6X<=TK>MWuiZ%d+KsfY-AMb|jmo~}Ep7Hi+(90kXzNF?sb#xXAm$Id zQ#f}Vf@4?qu*aP+7HFejq?JjRgEr{RsS z{crmd&^Omxa4Hm`{!~XoS%@JL@nAdszP&@C_ys7E=y$Q~lSuU>K*^m2o{Q=$Y)`^K zY>;g_SQ6o?u4#8MbR6VE&~0ow2*+BsXg&kCX*JuC&w;(0*AUK0RaRTup6ee^>5yz- zO>YQqd;m~+C~rT`reBx8dBIsIG^+*+fsImwQyr_KaUMFrwmH`~VJ@nH$$*w)kl=8P zEzFM8;|Ty8wVHR4Vv2X7tv)PQSbYhtE#8K#cz=5ewdH*!3krOVZz0C6{JO1eC0xwP zo9fY?TOn>$950?;Ij=9yHT--a)VcKYh3pCUWO~`zwr4WWgX5JvsFChOe#lbkr1?dj zvW)lyK*3iDS@`|^l)!2o8gKb4*S=x-D^&$>Q>wLKgvL$XythRz&qn34mgFdcFAV*m z4S){GP%F}W%C$#_WIcTvG_25jNx5uTZoh=PZ;2G9&JF~ynjRg`VI|&8@*FM#a68{1 zTL|Bv?L3)ZhBF;-+UW6OKjM*zw!ORr%qsSBpE;lY!TIzL&ZmEHzVZ)xOB=nBzLm-2 zIWdx}vHKMIU5@JsISchmj^~}!YbsHH{65OgMMR`o&2H-E9hq{moTc!H0xZd2+9HqD z*{MJrA1I+uYAV;PU`ci~ZT_vQdnYLU^ncnvYOJm&61_AK_;Q5)r8+;^7_IgJ{6a!u z``RtgC$Z@v3ui38wajKJq!%1lwqjLh;xPTy~7Cx74F;kHVqS z-TnQ=%hv<-B<9zKxfJ5NCt`}o$+B%wF(qqp`h-^f5LzkkZF@%AmjVwp0H#xy^}eC( zPzDU?sz5{8qKuj>Cqe8{MoqSpKotaK4U2Iw0g6c7qY zR{-P}hGoaZ{E0ZZX7|KRP0@+ln_4Em=kyn{++Bm8nPTtt*8qvx6*W^DcU)Qbe&deI z>i*ui!&kStaYs=dk5h(I?d{pGHhnw^j*Urha7?Oi{Ak`u4e3p-lNy@7t-yBsih2wR zR_w{&twnxMir$a2=s9@yZ`bjS9JqP|pLX}3WZM%sXEyG*c@;ie0b}asvM=<&!S1tX z*!m8}707WAmS`8C9^crw{GmQW~XnA(xeHd*30l2W+?i=h3h(woG{LxyGnQr{Zbsj~ z)kxn+Rk|MhQy6Ha)rd;i5s`@a zM#v13T(rxcUR4!~-H)-s5!bW0V0vw-WS^D@@J2kMtty%b$eThO1>JpFG9H)Xg|>}I z*lH!r!drSopTg>QOKJ7ky`^GO_yVbF=TlFIl$HD$NcTlXO;K?%%cSnbDx?Q*M*#_? zLaV(7)hNFe&dZYNI$oyyRf2Qk9{1wtM~5rE}E9z+R%#X%ly1)yvR@Haor@8EBSg+JoW@^{0(CkJjJ z6oJS?ECMeGM!@ChHl7B?qjZ9@CzDY|SH`IvC&U~5P0;@_|0NlAlJHZWk}tTzJP%;o zf`eIJo)m^K(aV!UQ4Ns;NQI-C6e|s8e92Q-s%dAJDGNiID49f^h|BK4rFjr7ctRZK zbX!YmdVAu>%7EtR)5O&Pna}-dKCi%hUI-q9&b02bI!qPoe03$5D%KU%6=SMcH?q#C zhEA$fr-r8YCm|k0)Td2UonB6F+CAyzrmrhD@n5EOdg}}KHNaguJ(CP)9b<;WOlDd3 zuUt6HuU(Z@EgI9NGT>81H+G+`@QHk2y#$zCck=_eXwk1%adJQ!G*P_zfPt9-PsP9| zAc`?JNM5$e6MPl}pOPm`R|0(Ey%Nk20@)X3q7-2}gyI38tpj`h{c+foYI>grJbpEK zsU^k{Hc9g>QWjfc9P@M@QksJjf`-m=L_7GE)rA^km{PVP*Fet^9<6LQT&_^iX|?C# z7qJ19kizfrX~N&_?VU}CujnK`hrE7t8+}8Hd@vAi7+i^}lxQ&08}N15?*6kJMz{67 zrh|+xBJVkddExijx2o}%oR51Y2xxJa2gU7_3dm8+PN{$g#p{#`cu=fPselK?>68k1 zNVcPU^Xto&AXOM$oUY)L0-!Bz{xllXD{+@bekO~QEl>7JLoASmnMfg_oYPDMLo<;g zL;0wgI7KCj0D;HaX7)!t@c*$sE%Fp<(#gSO)!j(KaICK7PRY=PNCen16bsDgS?&e`sXqw1SC<>-?Stf6Pd`^=8~H zwSs&HZq;fg^b!Dj_lPn$YBgRYLwnr_{KN3Q6}{GiBasICyGlh62Ns)Z6%)JS05;p% zvaBBObVslP%r3EV{D{SXluNR;a7uFKHxX%}G!bbcTeJNlO|vUPM_o`hCmRH8q0rIz z;>iaEKSZObLHH5sn4&#+DDR;Kpnk{V0{tQWU+w$z+(4g(h*0SjcP;Ef)sAIM0(PK6iznUZPdGo1u2E|eIRs3 z=_3#>bD>ad1kGl5qvZyc1*}XzeV!{*Yl&f6Rs}WwXt}Y`Vo~c??Wxs{UO$BW24tfB zM#z*Wm`s|Fk_=*-a9&wPJ51k_(mt{80J}e!{f6kipXlEEFrtZ+{pK8?-5MJ}_Z-YC zFyW|#_IU4L#+i#TNrGW;_6(s&$DSi*9LU-wPzF#Rl=7iCorU{jFEJwn@P_ia(G_I* z+(^ikfL<^Fg-HZy$vSY$RqL+aQ_-h5;}<9$8H?St0Z558ADw`^xjDI2khru3q|Lto z5Ml#rY$zYN?$%u7-kXP<<`BR4&A<4MOu45&j%g4Qd{vA528BUzaJ`I1_C)$Jrq`ar zW|xKr;Q{(0jP;wSC(hzyB=-)exc1zj)oy^yQs(jXs7VjMY4rcX+6bdnD~0ndI)RoT z*XZBOqq1QnXshHS_L;#ReK%7gL+JWX)4M(+3v+x#513hk3MptjBjsX@%;Qt>$O2wU zMoz({#5D|&5+%kkWWnD*3|a8^51Z0-D93ZbGW;4B)Z@4Kf)@Pty+99qiFow`Wsh^( zY?1CAmS!0FTgtLxCnE*U8n#Qv(U|++GSasQhYw{JxFD2f;RvuI+K*+P(^;b~RRk)GlO!)GlO!)Cl`{ z1WTdr%Xl%mBTf*CefJh#`gZpsyvm2s|KAc`g{B+$@GYt9!@03pv13SXa2gW4;a2bd zt-kb)Lhdu!qNMfzOT>u_afmst7jdGQw{26|XWl@oP1JSl7LYT;LQe8}kVthU-uJwy z$orBa?;01$`=TQ6ONzW}guE{*^1h_VyGF?Sq9X50io9!tye}&9zNE;zM#%f3BJWFz zylaHKFDmlBq{zEQ$orxmXa;#(wb11tFFHW|j)p81jjM(thGs2CORJ*YopOFvl6C<4 zrS$g^EO}~34LA%)4LA%)jqDLMjeoeZu4Lh|#y?zESG=&k@ejT_V_{3;ui%7Y5Br61B1ai(3Z90@a^C<|%Z2wabjM)`WwKQSSJw;d9Cv=tlldiJ2 zX!e0|bp!lm|AfCR$>;-WfcQr+{m_sm*gubr>8FN+V2Oi?Hp*G(MDA~hk(c?&rM4|) zUr;?jhJ)_eGZSMetCkqJi})TUdOX>7jgMGeQCva0h7ujGPZ+MH0wLe+!?mp3ZX^ABsmuTixQYx@2- z_4LC9`My6gx(_L<*%B-pWp-PiEvYTMz-d|S zs~b?=SY1>%6{eu-kua+kG*%CVS#?T6-n}vDJ!^Pvx4IM0PHsP|=V*I9y(v2B^`>~m zcK(Z@l@eJ-+l3_hETF`QxN7XFi(E2G*{}Q7(S(}2+M2o&_Ey4XkEsw+yyA>Q zJ5#H^6BSViE(fSD;;Kw;A4yeGCzIOLx+smTeOEWGo4l%IHD|c&;<}wQwHB}D+y=ni zVuoF<_@ZvF$`hU_scThv!VgpH4l!@)YT<=kbYL1%T_Q<}=Xc(dLE{94r1XJ;eEuw26W>h|nAJSczYgpDi)}qo{kmg3`r!9BaF~wbn=0bvgF#FFIX@zXpA(GsY zc)QUq*b;w-nme>fC}@Nm*tbD|6M3TAhu@j2*#`e=);eN!;|?@<)iPdYuBzw7zp90o z5v!UG`3*5IE!K#I1S zB3Tf>s`3F+AHT1POFb!d{}J4(AX{ z;0TP8O5^a35TrEgU!eXA-Vf09CB zEXrDxkM~gg2HmSOoeS|Bsq%*+`3UkS3&R%**8f=k%s>^L$sa?>pJK|N+aAcm5QM^| zKL+IwbTTD>Jd{7o6ZvCM{xDC?N0dL%x|IBJp##?fu!H<*0V%-aE<`ul z!YDiIK7`O>2x6012f=dH&Lz@U_7B+Gg&vMZy!J@_lEh-g+#0NBBAc?RptJI>^)>^2x4}EH@ zwqEesUD$;L1R-1m0Sy-=>k{rH;g)>g@0oMy;wbhq;@mxBFlyG zo6_5}1@*83zIy;psXtyQbX#dm4fU~oya;jl8vD2sPa)~e1Zgxj0WMdU zM~kfXW4{{D+CvAP{SfFg=((Pckew&$x!5nlLNDd%dZY_|lqc&E7J4X8)gvtQPoAhp zSm>QRO^-13&8TyzFUjtljmtUCYMJ}>T_dxnl8R5)dv1By{$qx?P@beEax^6sByuz* zO-(G~pfW4`)f8qW7bwh1o~r2mwgL2e1S*v77WM8D8ztbAfcq!=>FfJdq~~Gp19H7{ zzUY%>$dTtekk7DO*iS~!$Qb1s#DsVwk7ghy#2dM=kC-4Y@-PNsg1pF67>Eh+MjpRF zOc!&p9?jkErPzD&6|KR$o(Bu|^t~I4HAun!)+=9ZYpa+)Ag(l>#|yj|VA!Bc$bU%4 zF@Qu0z0n={EjLuscO0F&MSRBr8jhY~he~)PD3xbGG=h_HIM4QR6{bP(G$xnr;|d0Y zB8YclgBrKwPiEpH_^~W7^zfp$Gwb|ym`OLz zu_f87nOYs5kuTffmv9Iw*W!6o2u5lu=8H{vcftwRhROqqqm4VQeL*ty$wWiS{S#F3*twmWn6se)x93@}kGjiWESF&UIiOrLLO@4KwO>uB=;FEjI zUe0_A>|Qo;FV4w^927*g4%0rT>4GD-v3vur3bJ}^a@?Vq$;(nN!i8l)sR_SaoWL=T^@IBn5*p^Yqyf1IQ*1~tVB*=CL(-tT;5j@4 z&)}bc1m$M{v<2)_;ElyTMeC6l7+L7|+ZO!7TYTGsj4ZFeI5#H{$S(YpX|*D))?b{J zv&rwvdh~?3c+SU`^vy>C^LCE!hVu3vX#xL~@*YK<*5WU`5NQ2HTk%O$+~z-E%effv zJ=$z8K7~43j|9%{T+j<9qVG!+#TslcfjEm*6kvuWx*q00PQjLdcQS5BoIA8dc^><# zpQ|mNbdxK|TRdr~E2%CptYBC1)}fX)8%;>=ndB42EPmUy_g*3|v5UN{ia z|MS;~Jn{bxe@*${=C2h7e_i-p`74GA%#NKf*t|HwU#Elf==>E-H6gVEd=tzy1y8|d z`}Sc9jtwC-z3xfLW`}%?0NLyNZt)!D!!!5=*le`E=^s5mbpwCASSLy8fuE^8KpO{7 z-Qu@pPpR|UtW$R}v?nmQa0i)gonX3k{^HD>odIj%KBnyvX?y&|gL1y~`vyIF3XJ&g zmh`X4i0?6@&H^xE)(ol?U9*=pdr|YIcIz9kXt zIVH@V$6#hfv*#P{-V(U+UZ3Cg)4S1(pWf>=_-{-cJU1qu)GiL*8k3*eE)E_Vlb_Zu z4qh3P5B?Yv2TzQN4{H|(?~BRLX%`2Ni^<2gi;07m#l*qCV&dRgG4YA*;^0j&`GxJ` z;6X9@_`)%9@S2!7_)AP2JS8SRt6dztBPM@tI9`MAIkg9Q#NBJiC~A>|quoEfXM=yt zeTiEIbEu1Bc$Wz8iSyes@7orbx1b*a$1o=54&6YGaTW@n-*(qMTZ6|Ck{ly!ABy1@ zDpV|+tH$QZGgPtytA#q*^H9IwO$?r~< zk(O0w`-FO??W4Fj6Uxu9=J1xrVbSx`Gx{35z_CUb-V-aZadL@jTC1A&iA(BXN5#D; zNA0+&z(ypfB47n4jKAt(kwg^nxwYQi;A}ni{HcFUZFB3O&-@2AHn;f>oBg95&3@aU z8oOtFqJQ+in!Pxq-Yeg=!B%;oPoDB_o7VGlhcB35+uC5Otbfwkvj)oG$8oo|*#ppq z7j)yAXJzX^)zv!tFI;{Y!W24$zR+v=S;v{c!6ncBCX9|Y|6Pblgm?T;Xynb8KSsJ-F-aVO?iucb%Qnb@s@v zv-7&nws)PacAc$tojtMZ?1HYd3%kyq+I4nO*V)s$&Ys?N_KdExXLX%j+I9BauCvQ8 zo!x87395jtg0r^?74RnT_qjvQkfXnc%Q*eE;(Lw;|BjI0=*D_(EI&6m16iqH=e1+rt_=<8QM@}wUj^^Z zb8DN-f^m~S{6FT`so>XT*wPfktI0)Uc=dJU)tXweJDOJudtf_$O?zXW>M0G-WQ}%? zl8jnZ7|p10s3Vq9qo-JsHpl3zE|&@ZoUGd$E7v8_=185$JjYirQ?$*hO`H?yG?y}Cd=NBxaYFzK~?f=O>I70X|^|6mV|y)Qe!@XQEhmz&zywIQYdV4t`1IeG5z( z-<5L^F zZ3_H4bSN-y){&z|;0_TuyWoM-VArW(cCAXCZ7|t&DNL&0zq=*y{dH%7q_;HLuZRAKe$~SI$KPBcDp!qwwU~p?cx|O zG5Pj(ap-I@`C7X;bheoMf_8D}Y%%#$+r^98#izB4LuZRAKcih7I$KPBX}dUdHa-80 zn2h!w*A!1$5O?=6{@&Y!T+EhU_nf4z7N+QH?@RpJa_?V%%EsLS=pwYFq2#uHPXD>=Zj!oQD9^BbjMkA3gg-QZS zZ&9B45deh72CuPPNL%ah7eufG<&_OI_`}QZO zp20SUCp+U-&&ZGLq=x`i<7Ze?dCfh~2P*Ft^uFlYndfcok>@>X&WmHvoL8~V0`+UB zSc4-->*Yb(E)NpdHHcd2Gh@v(O{so>`WyARz_n`bE~Sd+sj_N=6Yy}534|A)ps%%C zZ!+4A)h5$AVG!HUFqT|^eIH`&i}r za`FQ#6LA%4{|r!an8{4P!jroku?F=H(m=Bc)8EzuLk=Sl^r9yHd=Gd;bKp~`URB@* zf#Ny&8R`C(z(<8`*d=6X`yPm;TrmTOJd8F(&qMOl(C=?6sKKs+id8F|jveVsFR9R>#CXiivq+V*Z#|O-$_5 zm>AaNqFTE>Cblakwl^kL9}_zi6KjZx9f^q@kBOa*iM7PUE=0vf_6SEd+;hTzP>8Z1 zb`WI`&g*+qT)=k>a;v0s4`Q!E-~gyx5d-b+yO#8>;C4C-Z0n;e&W7nib8U3EcEWS1 zr>_&zj#^?~q?)$0)!Sfac9yhw%<_QBiw@H|+Ce|ofP9`=)C!W?5k3=Un>4}^{`uptTTJ*%z(TV5u#Fps93wq*(=)_i$Xm&-?%;V5& zK6u~8vcB*?YVzW%1RsZyzsYxvQeA<7!L)R$yNpXB zgZ_POZKCwUos_OX>8H9X{mmi0^qn0`V|S`2Wf{f_ZkrOrWh916Q%4NyzSZj((XkG5 z9x%XQ!>GCh{?p(dQ{aDHRB4-MI*fl_82=i47ZVUtp1FmjUTkE*7CYa81l(^l`c{wm zI=ya;(`%Hc`T@ws+bY>n4ZCq57#D6`mylKaz_P5mSpKcNTbgvy)tu*T1HU~6e#;>E z?RxUtX0^?HiV&@rg=o7BM0Qe_064D!R*St9)ej3+%S&jl)pE}&)wd!p^o}arJNZ*I z%!8{Qc{%`Ff_yC&u*iqK1;$Li#HYD(uQ`szc@VNgac=GK)tfI?AWumQb=SN978rbxha3HT%d zZzZ5O!st$D{I$`T&`i;o3wWS04-)V;0*bSX?gW1raE%1qCIRyZcn<(W*oB_(N*L;8 z3H5ggbv2=;5o#Qu=7ymjlTeE#)M?ne-+zcuRzTerhMFj$?vqfP2~`LvTw%NJzpz#A ztvRi2(enMmBda-PT-zbLXJbp=;kgNM4M)mw_W=k+)ui6cTfg3J)c}R>S8mnBYRd=U zZx{!@1|KSg5UMRad&?bT$12r#C9=di6-A1SB1NAhA^1Taej2vUZf(Oc;wvg!5=;A5 zw)88z3WMUrLijtqQgu(Y7KMI-_*uq#AniRL62q@B@fAT{ zA&4FculI@4bUq(dS`2tmdNZ~W0)GR+;6eb36100pl>l1$`(6&0=qBDb!6?BCh((Ff z=;Ty(GJ89Xfv8Tw`d%74?haIA^!Kz=sZkw%D?U6sJdGW$R39Fn{5 z4q=mceKc-gMg995&VV-P4J#XmO_Sy*8~_H0Z4n4IWV~m_sZ@Rm7^7M&vorCdRQ(aP zR(hM^e{jx2!O1=G+N`!%z&W4iyYDySlP!M`^iz4m3df{}8-5AO;Be(zh~-f6W{JK0 zWK-muF=+T=ar`and*PC2e(>%Xrv%Tz1;-kCZK9Zjxd?OoFY3$v%Jo3c=Dr)Y;ICAy z)ixZ*SmkSRi+Je=bYS14x!@WHP@x|f_HMKQ{)AB(#EZ2_j@)%~mJ!Par~7^j#QXlX z_2&L45a(EL@85o^7$5IN(L7ETo7*fL2J$42XRbbFT#pb96cqUTt|ZOVw&~6n?R&TKSaRqJM|+$j;_61kfQ>mfgC-C2gp(N!z4#B zRh7Q88g3&%D(%*ZS+&B#@Wxb7^|P7M`-8L}5QPlQJxdZK_2Jz)=;eWdvl$Y|@0*uw zADDt(ABeix>jT^OI_UKE_^@FDmG8C=jG8Y$K=aYt17&Ydj2dn1tY7HC@Odj7T#%`ycQ{#|D*+R6$w5EtEDmKquq=9B z9hT#JUb{-MQ#=4QuF`LIiO}DPa5#Q=`*##F#$%5K=4gp>j0bL=WF^i>7h^GzN5r zx;{TG4Hdcb6F=k=_t=-PWTEEHBGZF~t^#ev9Sl8T@n3eUyL6}pl&0YnMmacMUq@vi zL(6R}`$n`wR2jNfUA+rq_j*)^%WNYVr9H>Erw+wXxEn;OUiCGAlMMvjM1+38PVvIX zE7Z(w4%1!|GuJm+bq%2zaE;@-pLh;KeQ|Cl=69`9bsk~OeMhSL@Y$UFtUan{z{+V9 z;f9|p!O90*1V~OE%8liZxNs+{&e42Ab>ESudhAJUYV#NBpfxIVIak>*N9Eca9N<){ zcskm@X#Q~f)hRGG<-%k1+0(wj*;zG5i%(v(oH5NG zo|h76eG(YDCnmXPL8Oy-0y{S8$=?7$+<^{Q)jbPG06yM=H%jR?etK)$x@xX+(#LCd zTv-|s09Nt>13^^Xj#M??hAt4-Ug!vRQx^RLg+7$M(4=~1pliUFGeTEmy^{B-NRo~l z$8yx%wMu0LpdCm{_5TS2ZkQ$!9_=m|rjB-(4r9NnuAq+cr_b>#tQYX(kPAb?zavPG z|7B~NZQbH1te@=^>v8e+EcifK%nlAdh%tuOXWN$KCrgq*ip2g=9)$giPI91@d{|{@ zOixSIY9=1|$RofRNDmI8zrcuthfpQJmxjUrDZ%q3c%lRkIRP_03}cos{RxA+1scA2 zNK_j9FQ5j%o5O%M33v)B5y1WpFz%L0Yq+0pM<^qNBbNyfAAm#7t#0jVeAMB^vd|}@ z5ZkplTy6-;g>ZFM!rn@wd6g1+nS906FlaY`;!`x-ur;5ciQf-72K}6&a3VxlSIZEWK=$p7^hVI3n4mPOm(=Qv^0E!aRM>O zJth$nkW%>^GBx)TNfSKQcW-r<9|nbOD@|zd0&0xgemDifU|A|N(wMPsf@j9zl!iZX ztU)uXZ0T9{uW;sHCwR&ar`jrSwv@eObB{HHMf}?Kkwv_{F?0p*>xQAO(ht#VlTtYo z-`C-uU>WBgIfzy7Z_q@oW6hzRm?uvpHH<_YfmG(&++!@LZ@BFfSzlX7L4E64-?|(} z?q_qLYu%cPy0QrUbweD=qDW{DD`+_R0KeN;;m;J@kA!96ZnL|eO#Be&v|)GMf*Tz! zaFAan0a1%MLKLRo4lIztI^1v?bpw|^Xb*rgoVmFFBouFVg`^GgZreS!dvPKN!*Fbp ziu(jWhu!v?)V1gmZom)Mhx{}oM8`AQ#dD(L?y-sO%NzA$rq@`Q*{=TR{4}*>1Nea% zuog=@{8*m(SR^qoL0$zuT08vvghZ_%AYQxf2sa1 z;aCfrN8^5|a5?t;G+9~Pl~vHqo`W2uhL4X`0s-U9!kS_0ncg<~*>sm0+s>wq)- z(m}EF34;voAm~48Ar^q?8LZI(&tS73GyI7#A3)-E12`vGNZ&`=LNf;^?>B(|0p9W} zFF1hv0p9Y9oFtjUL)99uIA9J>RTuD%1NbH2Ex!PRPf7SCSg4;HUXS~I&n0&UM_U`# zS7d)ae(C=F)BG!};~;5VG1%e9r78OuoHj0rj~Nc2aW}3|5lQG$CW%MsxM9rMByt=G zJ6HRSKK^^O53jEJ7<;-QP2RrU@9tADWUFDc?P+Ji^iC1%8!YGw%D8_s0j$-{tM_z0`$A zD1?b^Ts;)}2k|u|EIh})n$rVY3T)UZ%nKD-G|yuCX-Ms$UU4ltq>{^I#p4ey621EcyPP<-HG z##ISd7@^+DMyRi1ga-6BLh0W#LPL5Qq2WD@(5M84idi4qDnh+YMyPKank~`>Tr@)I z=Z(;ivqormvk@9~1|iXa(|TaQDLpX!BmxgC8>_tcvgJ{T5wL7La^Ant<}bHYn&k6& zj4;OkJ6=BD$7o`FV}g9{!w6#h7)}k(zX4Y$-%>MHdG}?@99Cz`fnyfiyhk<`-!?z< zIr9hp%B-|gX~Xlk-Td*kG|lqY_&{LJ@dST-79t;-`7E$(N(j6&=L8Zn5&5dS ze4gmR{pEFmA$#@)(pzzCsr(mv{I-!e*o4;>|8#!ltNowx`-aS}GY5vW?!eabtM=eB zb!5$E)+dC=k&%&8>afLb8itE4`_8@LqA9Ng`rr8#3)RU&tS{vi(V9}xnt6|GlXyI3 zlFt{>0^;{S@$&f*kY{|ej(5F|_sIy}I(o6#mnY4zVgp(BonMEs>HiltA&_$C;izVa z7XGDOGt$0$GsK~*uT87{pUzt@}b zcfA>K#3_c=R=pVq^=6#;4$WBcpEjf7yElVQxucr#wBC%D^=AA-Z-!TIMxEY_LwYls zF58UPt%Cgj-%k*>;mh!x!VFKm)C`{&+XlDZhCk?SctdZ4PjACEy$xUMZD{Vg4aDPc z7!Q4Jo4cIb#aY=IckT+8NqMC>%Z`N3>C@w}*e}(we?`asO&$9+I`-Rj?7z{mZ|NF) zfy(|cDiVEx&i*cllPL8?xRl<2FFJ3)6R{2O=neRz-hj9C2CUT^utRUaVZ8xoze58K zhEa)Xz`@HkAeujjChzUM0p+m`ct&r)3cUgU)EnT}8?aMvK!e_ZbKju>?U{o>=j+Qg zAbR+V5?^-Sfcdcvcvf$~O1%O9(i;%a8}PZ_fS}%h^WUKX?Pq{Mrz`$eoZNm6Bwu=z_0ZN zys9_g9lZhT^#<(L8xYbPaPd1dpu-#>I$h2IHXvpWutcYGV5$WyBg~SY)3JX|$NpU% z`x+hlFLdmW=-9V@2kbjc0OHc+1R(Y?6M!W;od8oU3!||AjgI}Fb?o2Mv9Hy!-=kyS zsAJ#u9kA~(`iV&Iis^%c3aEm+P4SMaO)#j`;>1^SwIeM+N37mI&hQ zBUn-GY# zcPQ`O2xTPewr+vOJ-Y&5x9$#HfS!O&;nq(Ad#NlWPgxOYq?XXwx)phKNUcNaBx-?K zxAlrHW)$#tRv;N${@_~({_o&S-vygG)nZ}VAVOh#R`B=lwnf{&X*H^BgU_Yq0QVik zo_Ak2)dcrx<~FRPdlLHBb{875KTl}NefP@efYW$#{|U7E#2cx^TWJJvwE7p`HO0Dw z4LpP;uo*XY3)J#F?!h~9j+z2H_5=evE_@qkotXs(+Lnv3gIF5jK-=6H*l{L=)+2KF zYHTO*VJkWuRpIE^AN>c#g|o*C;Z}2ALXEt7_FTW3z&V&14C_s#ebM%SChGp}>;X4N z*#o#e!}o1`<$-#9+~;Tyc;#kk4|o?(X#Z%&RefiViZThn9?+{sw+Gy3WK^IYfEK2o z8mViy<7Jj%2Ds9Q=7}0}jA)vOX33~-2Kcd&$pLUxU#k)Iis%3%x;ZlBz^L8`NPEEdj8N~MMyPKOBQzkv2&Kmxp&=$CG`tP%619xF$dIrHoHs(f&l;h= z%|>Xz86zb6M3fkE!blr_%m|HYL`bych#nXaLLh3q3VXnPvPx%)*{BK*RWsZNz22^c=>$CET6Rr@(E*xuuQ5aR8O%0e45>{YBOD?Sc?MIJ3kLo#l#n2gSc}W z3)My}4T&!Z_O3K8F>a2ozFjkqVxGKC_0}TK+*a95)_?JX>6zk z-rJ?6K;TV-M;;9vVKh=+(UAu-9T|GN7NmXm7W^`{1(fXd7CfuB;8nc^@98aoElFTb zGY?xJ?F=1T@S3nL{HKlRZ~g9Vm>$~(3io;&mg#MHO>e_$y$zr0ZJ^1AZIC90j%}D` zt^ZFOFxBe)?k(T~dDJ+fbg#GI*Ln;7thWFbJ%RBiy#=)Pum#e(&{+#s{O2vG`0g!u zD7FO@@AVcur?=oQdJEu2Szt`F8!@Muhfir(=-2|e%jQ38L`sK=EbNROCo<0fM16oK zBk2D|M;|6tfu=NhY8IN?h&IhTd`g=_d-P$!=yVEqlxm}KdVEKF#4j;$01Jfp|2B6+`6`V|-RD;#E50H0BdA z8jty;xsp#B%lM?JhEHiIXpcCRxlYD-^5w=j%f*awmbklm>#%5btD+6OAs%I1!H-<1BHBF+MxC$GLn$glMkklLlfwX`bYh z#xXu=n&DI020BB$lQB*-x*X#~JZ6lu#3jbKT*cC*G0nmS6Cy+dJ)g7+^GUNLpEQc` zNs|np(k##!;+>3fqS56TC*m<I-m*{|J0^XUJbN|EkSW>^%uNI{P2O z8n8iF1J=MAumXymum(J4SOZ=~3Y0|`{(jre8gTM`-5St16dH`Ney;2PGM|TUH z*|P<@L@L(*{sa@iQ&4&9c9&2b{+< zNAh8S}8AgNA}A<$n_t{}d|z ze+gN{Ru=s}4P2bwP>6f9<|S;ESE!%sw>5AQSE)yt0cej98v(8|QLz{Pqp{7^(|a-; zs8hYa`6s%a=C&cynlT=Y0Sbi!a*@D5`be2Gq(C-z_zcn9NdKX|J^1ZLE`(ZEIJTR= zcvOB3Il~Dvv>fPJqw?X)a(>pm-HYd>XQXdz2@G%j)bC55y$zlwhh(JhZh>t}C_}Y@ z>?xlG%99hjlUr}lV_Nq-*0IT`Y#l3`+((o(D&wMZ@tjc^>04W{Fe@}dqcW{~8f)_g zKAM8{@8t6CMd;74JG1JWK>UPFC}I!##al7sA#4xgIcUQkgfW3}u~;>=WVLa-#DAe5 zIVPUP6E>ojchS9HKSs0Gl=K_)4%~)Wfc;0|@sVxVhsI6G7WSbgjEnLcy9eH0YIHP5^!r4#9Gi>mNPJQsZBfZ^3nC z!_94hz?qAf8*sSoT)kem9QES>s{SsPBkBLg(r&z5&jw+HUP!Em*Y|vhPTsjTYDk-mK8DwH=AGz>uFw@_WkIzdYN{0R9bhn@@Rz^|G2C3B~J2) zKm8x-9G(&2#ze&7Sm)v|8u8Z@*zxdDJe%=%D6r{a5cm>&`}==}`TOK`F^)!D z=(zwc$br=j-bT{!TizoNztiAjzo5Vy&{ZoH?KF5PN{OE>A&dpw*c*s_knv+LZUKDzuFD1^b;_tXSa^r0W7 ze?!=3#}X>rOl5EY6Z8R5o2vz0N?UF3TM@xEte(EDZ;654`x~4jm*e3U<-MuB_KIC? zzhk&3!r0aJJN9`UJzY%g^(7zHUi_WW6Zc2nZ#l*A$ey%`gNQoKdje<9U}=C01;1Lg zOH>KNI_89|lHn<4Q6;2_yDfg0&@G%?W5(QVIv(0hvEr8M>BXmOX6xIm~cPy6kI_Cglhw>GYh(P!|zm6H&kgkKx`qg z+-$@5Mvo6gjY_kDV>L5 zYvnkeA;xj37{|OTVsF7Nj%D1meD5v{Xgy%p18{J5)h<(D^}T!GcRpclyx&&@u*dtY zg~LJknEv3N-F!W0*DzcoYK+^aR_ePPw?`0?lf4Q(6({ANAK_e-q(^BWaM3=D23Tbu zkCM;6XMU(Nw9|SZ{d(jClkkD$c;+x@()|}{aO6s#L!SySMvz9{?%>6!jn`Z>NEiqx zfYGKD^mq+(7^^j(PN6FzQ64;PBib~$uom&{#dDk)>DyWYuNOAqt603>qc1z%RW}`T zgVWC+)W)n@k)>bMfvoXxtwLW$7#4%PVTVRdz~X0tk6Yn_cK#qc^%%%E!;o)xfLt8|`B50s z+X2!a16dP>{Imn)mKezGVaQz_Aos>V)`uYvb%1P$fjkn1l>HdppQmFWTf&eR+SLiq ziTo(+__}+7{GfRM-aZkBS9l0$^;;vx7N-vW z_X5>*(tqH56Wp3KB&rX3C#69L#H~n~z7Mhr7hJ@7!6<%-JEGRYREb;cn@T5WxlN_x zc(>X*Re9s2>N`5*!6uf@Jg$OCZ^yBvKaV$#O}z>!2=c<)cz`0$n1W+Vqizz%oUR&< zV@u{N9`S2Zsy1PgYcBZ`hx^?MMENfG4gIR&X3%(~Es0M1u}tH_b{OtTnf79ITD(kK z5uJ7n#0B-OicZ^uG+X6@iE(APE_}hDK2jQ0$ar=TL2qZb_G_)XW7sa zS2K>8t-^tI_Z4c%8qHO@*5YuFpNmtJ=%u*0Hm@sQ#TlVr*zQ^DzTK?X^FX+s6x7p( zm1bE_5$n0#EGj|=V2?M-kHJB7SpmB9PN&hG1rTJ=o%iE`?p%vH-hWVb=Y|{5o$vfa zoEp0>3~!b2!zKJfgfEitzaxA#;DtOj5l5qxDky~*0>|CsC$`mAo*ktu{?LpBPu~}= z$6HGhpH@v<+xDyOH_=4Ettku!SU``vmx&ML@)ZU@8QW>{FMru-a?%r>CZEMgx=!jY zhE&=~^2Xmsl8M>*KgY&@fv;^86ZwCOjc@#WY<$mKvGLG*vGL!-8GUqp4|`+dKdy<>@`TvyU5-j?SuJwhswt87t=|mkj_hd5;e`DQCDPOpL&tGA|6I};xru|ufpqq z8txbbT5S^x_=be{#v0`L9mx2gm+SBVc^+E@a=Rl}aJwJl4!VYZKN945aTvZ@!oMKl z$KM9{GdBwO^FsjtO{RdK9EQI`!apG4H;n-NixPe#;op_;*N5TLCH&7M{2vJas)YYN z;hzG$l!v4I!7e<)JA75wHq||TnBCn6z7syVHX&|;yZ>g$1au$-pM1NO>et1n7?1Og zw-0W;WX8KJ3ORlR-EKtZWZHPd1C{ruF=qB&3$Jo+5n(C0c5wPK^&sL z1_YB(!Euew&o!7$@SKH0YxwfH2(07&yXro8sWF9i0TnuM%EQr7cxQ6YMW3p9bjcvZ z8+V?nx~k-T!2rNpXARTbMLLjX#k3&={VkN}(;;^%9J9mU0UT-6SlBxFdKu%fo7!Ho zHMh9dC~s^vZL#Op*c2zOTdA_H7M~JT+u>4%Wr@vn4L(iKOyM_6?h8?>kD@j>4@9pW zLdcnWPO095fI6ZS9(}8~Fj$G(@_)lPa#6 z7H=={Wp0H)sy5dVp?V@@LYFNU*IZZbk7w6R$EZRF?&MAAD*&NE<9EGnrEG@eR(6=E zt9duSS`sV))yTq7uCi9(IW8*hMI63_n?TY<&xh`g-QUPpq+uF3Am$=?9N@etndk3_ z7y7e^m-?oP_-DL9mBv=wv_fdAneb7i;nLYzO4T_Kb`2=7I-(+S@!Z8afxak+XN!pI zb)x?Z*?RFpjG(z$bs;rI9$#*)VFBDEgExwCVzOg=0kes!iJrksRkBLcQu!*}CWGPd zLr`>9EDEFZMd#syYIa;)=>z=Sg>*|XH){|5E2V0hCV%y@AHN~`fN zTfKNXD*YD}VgC+p058|vC1EO;2O{N|Wt+s-?V^6=UWW&|*6pPfmdEMuZUt z9#vv=kbq(QJq%*^1fa(N2(F3{0E8fRGliHydZkodEof+y&13J=_EPg%t)vDR26RXd1f016rFs+s9II?0 z$k;H>9YiZ+bP%HUTue2^(~rYed1IQ@;((`M(9J>{$=O!58NWJx#cfI;P#jb?ot}ZO za8!c`l3KYwLl{25;-2AY!4Sj`WL!MWhaBvk|bl7v{P`Z<(i&MZl)swCONH<(LGBI((3Ik#OO=_E@! zHRJFQ@5FNjv8@v`p1q$MnsjM;<`sBKDzB3?of(3vZ4sFo-VUBpfr!J^s8sdh#|aZQ zd;xx`UC50qE#~#u%MjwMUV$h4LV@rk#k^lIR_Tby)U3~NOfPzId~%RZ&oEORxK9fs zE|2t#FUkBuG=?wZr;XQu%c7yg1(akpQsB2r^*k=97{0d;-}p0fCr~kdbma|P+b&4h z;}H@@PN!$2p=^YHfuG2Xj|?IQ7FlHwvaKZEbJ@IFY=m*tXY#vSrb+X%9?Or@-3wu6cUcS95`Mt zV!^g1SuhP&$PI;Hw!FmVhnjwcm2B2Yl| z^c{mGy*s2^_&RiBBXi}7F`KlV8mcJvHl6p@oTt0nsgWY?NL}Q8A={mvNQZ;9gZ2ZaKBO)YZbaq4aUvV)Idwm$NhTtlRv{LMNNFnbXousjj|~BEMIu8avn!SzjC~vnifV zgSTr#JS}9H(-EFrd$QJ4-6a{PW~xR5Up?CIbGIv5pP@BO+r7)zKYRJqAE5DI)j>82{nX)^bP&8QI z3cj)VOI9#3!VTCJ96bg+_i%azGU|;anOYmw(<~lIM6EKaE z)Cmz$5Ldl>Ut}*iT$>%0-?XyM=_K7^c2WKSJe0}A9u8BjYC^a3i-5DR&sEb|xYy5W zBiGvlzITJvU*T`x0cVPohL?&tfTEP)TlgA?;^Sr#lXR$)70Q}ZDJWw zc7#T``xj&hco#`CecE|?#d9D@xmp9#0#hPd1OVb9{m4g^-jMq^SG7i>P69;LY^v{2T_0@&*9x*TFuvOmJ>@e*{Ik?CX zrHYWZz?=+cEW(N36!d_jZLKz9s#R609qbQ$E_@Y`nI}1M+t#ZiB-7!ChDd2|$@20m z*98XvngULEzXANAq5+})27iN3#~{z>j{L^uuMIS7X-+XH$ADcfBPmteGutBp>AvO#8FF15cBX|xb7 z1esXDre%H(nhupu5+qVEY89Yh`kbUnNeVXJlL7%C1rE7Et}u{m+162d`yd?(;hTOR zCB@Rf@aj>&gELQZQcy0{wOhrE0pdZZ_czzy3 z!obBa?FIYtinqXsz=nwGbb3W}NsdCdqA#N7GjU^vaO*5J8>b1Q?=?XIou&&$Np-|h zYXtxW*;1+sK-F-;iHWrFO(vrsR^h2sZ@_O>t*vQ|pm5Vm|3F;$3h>qSr1~`kC)sqm zpA+Hu3O48Z-s>9<0S^m};GG}-lzib6gQx(f@}JjB)k9QlF?vWhyxvv4;0(9pkc0F*iHIHgpvRx70T{ z)W0WNA=VdTI@+}n-L?M_+p$GOaB2lXXbewH?<7(PJHfRq<6KKp?96s_s7fB$E(pWTcJbg)|fV7hdeGu zxGvJb{D3*ufF#^;4MV;LB@kBIS%#Qhb=y(W7g>Q$C@F!ou!K%s=D)zuDx{zlFh@rC z8x=QxHNJ!M5cAluOOk~wL6QTnh4#+U;@v_l8ppX1|GJoX)S*`(7uh;7z>}4g-sf~} zraiKp#=;ra=?^S(xZmUJph8N4(lZiL_<@EU$UJ0TY zm8Vw)o2TrJ^#F;=Phvhs3S@L#b8(z^GtDS4Z@BKLA$vb zN@!WBp@8NI6|4+vVut!9l(M3-+u&{wR}&;cbrr6--~T~e-0G~j9uNjztx*>3y$mQkyXj*9M0(PM6-?gdi4b9%2m`B=zV~G19zv zD$l$W!Wcqw1!9Okji_#xds5`+{2yVhMBIV_)1Or0y;Lpc(qxBW{USR-t6I7l0HLqK{DPxfyn(_K*y{?sg&${Dn4kcF zSW!$X3=M`32=t@-4-`)r(p#V*Y9rnjpHrzyfqXz6G)IVmP<^DsV=q*nGzbRlN>rFr z`fqH-!GvWE&>?1_fAKxVrD{DiTqD7pB!{a(q=R9#Xf6tY7io?+p?f};*Om#mB60s( z513xpp#4fUPx%`ipj3}R%;*HAy2upix__f}l$$?C+eGgLuLrEIF9?0d8yXPduXPS= zgVlg6Ry@}!RX+x-)3p=A%^8R3OQ~K6tMV~{6hxTMG}twHEgd%8=1P3g987=zTHF_x ztK(aiZ{S+?V*}5!>x|Fa0mo23$q#`u`8^Sr{65@2VN^g`@je(aoF*uWke@#nsN{6$ zx}S%;?y8usvqW`W85q1AaUDgq0>}B2ChM0)?b5 zbffrQ@cK>Rp#jx}4cDeq?2;C~!E{`6t%EwiiB*pTrOFEAG}kGh^s+4FGB};Fk|`}$ zBsc6dK`FCf1~cRY%sQ$~Xcr7)f*3{5XIkp{G2q8rU(J;CO>>>sK#Z8Bg^AJ47#RGZ z>=#Zs&rL!xfrX!KAuQ0uny_)7fnF6U2?0bOLC={UvWMxBL=6;WIxzs9z#e>MWaaE6Iy6AL$4%9D-B@-=PEJ+|WtbvSmXLrwc zW&cE?UUs7{K<5!&zAbEjQOmM)5kDi!Kp=4r4T3$vC=p7bd>5flxIP&C>OV#?-<>@$ zf}&D=3fy5lb-{lc#Hu#<-!d3{97IcgiQvr3(=SY<`RPH@ zqh+1E6ygUb$I_Jsy^=$M%88()YANUyloQcE;?oLajMV44sPDzg)F&;^t9eZnEzb!9 zFu)*$(?0Xne$qin?Y1Etw$H57GiaarfMHCXRO60~5?l`gTMhRri^Viuyf2ZvKR8bz z;exL@GB;z+K$sIIXxcK}3&rfJY_%xA5zDj|Tqem2!q(Z+u(|1sTKRPgE^^5zU*kGe zc@UC56IRl< z7Kj71QwGdBZN@lkv@Zc|4qeAJSAh+BU*|w7gt_CaOPZ>E^=LR~xPVgLST6HG=T?aK zE!U+p$-Pw9hAeawbOkZTS>-Z-g?=jMt0v_BC?QYJCl->LZP2)4_JAOzuUjj`D|D%k zYmxCE>96BrC6E$@Xklgz?4QukX!?9? z#Jjc%DmI<4>UdA$DqL0sp$GkW0Dr;alRGlAwn7 zU#Absdsh;F>vH@q_@Se1g9>4;a;;g*-BxzcnYq=OxdT^_!`XQyUcydK(lTqEo_X<@ z0j*BxsCe#lc;-<(ZAEx9Ws01LJTwM9^gW)H!W&jTo@`xZL&Ati!n9Rpf?N@uVqpqk z1X?TSEts|{5xS#UgC2>qm%a;S{}Vg|qD9rM>KvA!xX0q>S8?`-cT{}2j|3pBx_ySI zE(r0WUV}8OW;lidKq%r^Pm?rYgJA`A;O1Nmv()g+H`F)|J@-puB6$+uB#dB%LPX-= zMUSi*=h3>yv!*Q^kCm-y?P!G0mR4(DB5IOT&fz-ks613J35Lye##Z??eCW+85}TAL z@5~+N$;O5Nus%ogK3tf!0~M1p^zMVpqw(x}9iBMGGiAUX*p0|Q`jKNYFyvM}Wq2I| zLO2Q>l7pFS)NY1#@=MA`5WnX`0+kw}jxBv23HB^d^E1 z(=H-B%UEyZJ`D(rp9j?i3d@S2^f>Ew9N$Yt1$h@Ciceh1DzawDhe&5pJj6*g8}wTK->WerZq z>Y3O$)q>fITe*h=X=0nEWuAawblhQTlI1ziZsJTFx$IU?3jDG?o;0#Mz7f)#yPvs0AndHX|E??GPTvQI*S*5QlbP*~w5A@s8CQA?Yd`bVIsI*bOzIbJ6+~nFY zXyQ=6EopLNxYoh_0&|n6J^dzrOP^kWU+w8F_$`0hi{GhFA1~ftZb`O3-HO2Y&S3%YInfwo{v`2XT&UaTtp< zQoYzM4p~T*+m^2v`9jQv=n=Au`{TrRk<+_ao7^rUt#wLuJ<=kzNsAp8Nsua}G9R;D zUv${57rfyK?1#s2r?@tZKnSNHY(+RMu3f2TP+a>S!!g7fn+C$-njqCHKqb`n;#!1g zN5r-Fb&3tT#9i1_$D#zHbE-$XNvK-8SYc&rQag(6wWZbAm(YYKL_3HN6zvR34Y`g~ z(Ba9xIYOfo!!&vWuE8daJ|Le1?m9v`jFj|Q2$5J1x%pcPbbF`+7KPFTnHAKTdL>Bn zabhJ!xu%`$l#gT*=rZ?#LyFb)bJIDts)FE15{;2bUL^OqE)ylN zxi&PEOc(nsbRmr@tC9V-w5yqRQoqCKyF(D?mxT}sa}et$2f*wW+W;%XEAg2MIL+mX zm<9@pcpi<7f?Mj^ExCnYD6W(8-a&pYUT2pFz1^?EqyP)ybZrEd%wp&>8$c5#Ob(EH zAnf&Un$C{*WEVQIkX3mf$gqaxXB7*8!lnm+d!4y8O6AyS0*&2!QT9L3XQ%6gqj@W~ zAn4Tj7N=_sXzEUfsSeb+P;;#XbvF1J#G4zjPDk?WFyWq<_1#FGgMvJ-<+>-tBeD0r z$RP;zVDy^9wcBZs>79Z-MC2#bchdu-T_X!`^^EK;J`mQd%-ykOWiRX|;`xX}62|5v zk!bELQI>26DFk{;yTo3d6;Bk2KxJeOa49eB0}0UrlLZdTgP3#wMUeM<^7}ZkhlExG zv9p7+yu%cNjDg{O8h__O&e!0t7R*TMe}Za*c1vo$3Dn$gJG-Esjl@~=d5<*s$DHj~ z51$0{a}SyPreyzM6Vd@8!`)=IJo3A z$dS}_9m7#XRHGS?1O^iCR)`#!kKn+Lk`s>1CVa<}hV74x8IrC`(#^-)Y_2VE?SQkW z2e5$7((V~tw7}d(CEv0_ELv!H;OpZ`p#IMjC)LwGSv7@JPs(jVa{5*^VcARV&NrrF zn(@+COxv{3Snf0>=@7ONB~8A|d8x+I9MVZ?<{$!%kJJ0!o2v*)*cVX(g=`8jG<2=Bhj4wSfZKn!swP}S3z=_+ntK9UCs|Ep zTl|~>Ft1d5xj(?7R6h-rLmQ*$6APzeL&!tewS=XMWf0wx?CJ3BFmHf2js|%9ZRc>Q z8BFs(j840Noj^z%6rBcVA56P0l4iaQ+;5H9Y1-?^{k-fohiOL`lPaz^;9GB3P2YkZ zhUatOToTk@@$y$;{hyAH7|={GVs9ur6jVf;!b)21mY5;?BSiHfdku4h`0BBG&}8s0 zVW(Lp#vj6kYRPfj{7t3PU}BDvvyRMG5a!dyptWVfnsOYQA_=mBwSe%$(!xrjw#tjM zJ{)gCg~hlaqtv&_ce*hWdralVNX!;kUa>LqS_YU2gFoZ2(aWH=xyHI0_lg$6k_i*qO%Fg9IK?v6 zIY(gH)>pmsO)W|leQdJeD&hgfm~Dg=S(e}woK-qa=qMb{oXons9oew1dPZK&)!hvz zFXG#vjhgA8qYW!A$Fy7wZ@C`MZiLF=aDB;_u%lt@LY2xQj+amx?59#M>jYvDAh_Fc z1xkq}@khmy_%M8y<5b$PB>pIu#6<*4;*ZkiAR=N(oXuZ?qHIi2csv^W(M_9qq(yH+ z4>SQ{!4Hj0u7oq2;san(+026{E^-+>3M5YlNi#sa0ExH2|N3cIZr9apK$9a3oBtLC zr=++Ae-&IrbFbvMaC!!hbecZJt>!XWq{6Z68obZrcb%Z3t_Ve zJx9^M)h}o>XNGS6$^-20V9kdG@nm}1g{~)kQo6|R|Ey&p*1g`M@38yFf_gEbX z+sFKC=HFnyV+$YJ9raY-==w=m>rxhrC>-vIsQFn5bexZ=cfipPV0TSEBr>hfisd@)PNTM*w;76`iiTd3d3=J#n(&plUmST&swwqa}YM;}EY8HY=Gr8su526GU19p_#R>{*N0*A{HxKn>g1 zW+JMS?r8{OyzNuPmI%-RS|3Vk9j0|Kp1}tQ$D!bWoY!1}3a>B9!5%LT{Bgb)fu7>N zW9lR*lZQ>+S>9!gPc(J_sip?*J1KJT#_9mMY%0e~`|~#-XvX;)&4pPGdBxVYodyT2 zTJFPsFh|>_vYmYWMC_*`XHVs}BQFMrila&uc$O_@HU_4b!z_;m^#(n=9#5sZfKz#T zExN(AMJUWROj8W|S#ib;d1f4PnZ(I`6T$*jhbKP?9f6C}2QNlvZ06j_)2ICs&J&zv z)x-%dAa0xv*do>BFNBp(c_iqVtqBjW6m{rBf)DcW=I%*_(D>rx${>^vY|ZlXN*C}X`{q` zD!i*g)U8W@q)+-u6>wbgKk>$C6shn!^ng-T!s&xPSi)SL36gLOW8+warcKdEv>++u zh>@@O**)2**xO`xox>r+Hk$qf>EVD8=itaTIBM`_yp+dtRi)}SF=xd-A901d1Fits zI!rf|o70c$ws>3cRlGl<@2Q@|VUXj(^a%EV8|;(l1oZ}OQoV3HN9$7^Zcj*gN;nZT zqUL8L&~ZNcPa^3d&5`8i8Q~k$h$g-gxR6}%JV;cs{rNQfPJ2ENzuNOflJ@KLVqbF%JFwHm zwbrih?k2$vw@+*t6LT_2bSz5diB;eAwzf~p_RE=I>=6A^@Fgle?u1vmcgJYUfw!BN z51FA^^!+;?aEx$&G8_>Ur(w8w8@}Iov8~}^16g#!BVqU%5`Mmf_nkN6cJzS){=gBy zZ<6pgh2cj^c&CK_HR12NM!^4*@PCl-eZugG5`KV$-;o4-Zj&Yy|7C)K|6ZJ& z0sf760RKM|{#J?qZG;~s;r|hae_g_VEaA%t|39++YYE?5!Y>HJmrHn$gwH4ZKH1)r zA;9lDEc$1B7+#a`cT4!bgx@afUqkr6OZb6d`2G@ph=l)I0sOyY{m&78KHy`%cZ@#> z9{aSr`(dATA7$}<12|B2uKsjBy@I~FMRjGv7Y3OH1!|>oWdbS8ElG|Mw8!8$Zeud7!$$vCydQjxJAHL$iAj}bKJNQ<~pVNI&4T3LY`Xr_%Jg$1m0#tqUL@y zN6US5n6kJ;#B6y^?5SG(lz_LXhKNsxgNs)parLKdD1^oh0`hm01-lWomsU2BlM#k3{&xZ+%wyGHRy(E(lQ9-qVREJ7fU zfkFg;0gmPdVKQ)zz-4ivf*1q*_SPTLHoAehkvOlX*wRPfHqbw>-2EKLMxreGu< zb3ddlG#F9k;cG~~DQJ>UdHKu~w$0TDYMyCt72(jYzP~`<@Is3>sIrtIsceBetYAUY zW=X{Jcr--t3%RW-gDaO1dV#-E6-S{Q+w$Zd($cFCrbyI=ZE50Pn0~n$$nBfaZ&qQP zk*kL#yRd}^QV^u{2D-h*QJ|i?RbMbob;);@_F#4ArK6EkyrKGA73;gSoE7olCRukO z^aup7A|$*6eC~*sW@`*Dh&eKh>Q=E)+y|56fTIwxmf~>jL*RA6(vT&`4gxRh29vei zvb2ol@O2=J3I4>+W(V2vde!XER_kgj3l zkH_Al2!~AY>G+|^f8a0DzGMY{wIyEs7A&d9Z~rBboD1zsAx@VTYD*zb&nPTd3UPW` zVgIERsznz305c16H?!a?cq!LY)ZEQw|3fnYq**Us0##-;n2cecw7IqlGlCCGbed_m zxS}0=CwuV(k9{}Z?gb5h?e}pK$2nLrz^HW?#Gd~3h0awG9eS`MMwYe#% zUc>2|OrO(ZulmSo!p6!*G_@K_2KFBPvzns$BkU>mHxGg17RpC59wMzlrtyKa29b6| zrtyKaBO>j%OydJ-^t)((^R!Ik18Jv48r&fB9eg0IMWkJjX?!5<0@BQHGD-X2Y`Qqkl_mFC*oi+Mtfxr z@OpA3FX z#!=aVBf(Fl#>IiIdp7XAFC=^mB>Ybp231hqlR*27MUzP&f4Us%o}@Bdfgwlo3K~0E z5Xrd+01CP_0sz5E2PY904Myh(fg1s+y|95H09yjJ7X6@+z75I!Z_Y5aWV0mAn8%od zcGpgv1$Mw-OQ}&1Z-*xj)>qPJ5N#av9V=o2i9FRaTExKU2B>#ISp>ycfm#idQ3(5F zeJWPS1(gsvy7@_|TH8Y6Sz{2-UIv~6{_OG5%&wE!YDhzGx9nu9?Sm@(|V_AAGbR}vA_-@eLWsq zRmyGWFf$TAWN_>`IN4A95EjrxI8ufk6poZD=k?X{{||NN0v=U$_5F+li8eZkjT&3j z(MFrJXrt0f3T+b%I)i5vE!ChDBT%KSZ$E^jq9O&7DC0OZUaDxVMQbgsTE#on)^G^~ z5CufVdsNiKpaLQUH_j$he%kyN;KKrb*&p!L?z3yx6?O6t?HRMC? z_#DU`4sur)@1If?pGE0t>d2>Dkze*Ef17FScQmQy^DWWaOq&_mvfUUmM^+LL0{z93B!O`$tXRqCz%BhSpME6ID2#&!Jk6c8P9L!oT*A4{il~r_ zD%y_`G(lWjUv<=@t3M79dWvpV`DPH-f=IP0FR_~%qo-u6fS#IGz?5SFkz|HAEn+P6 zoW8gtxgI8V!M-N!8F z7OmGqd@MT29{B$k7R?n--4l!M^P?|@MXyr$@c*Y+bm_m1MZ4k8Mbjo!dDm>tz#V(y zP6vgrW={;8fj1r0jy<8e<4ghNUb81mH+=cy0ACLOm-uq(7sZ!`DyL>VJ_A**X@3(` zS?OrzCqjM=$K`5?pA(5CM@6Kiby%|IAk)}Jw|)jt&B5(`fHF1YR(IFdFX^sRi{r^D zXyls$jKp0{MwfNhA?>Sxs!TK)fwqS=QPB-c%vCD`V{l4;veH>#iQieN0xWSOr2tDT z+#oC=ZMDSVaKrlts390Fa0TuODD1-xy$I$b$7b>}u~@5Mu~6fYo)GD7DszW2EReK` zeg?wvaZqh(4AcV*f}Ki0png;-s!~-H21y0jp}}?yA$G7`Du;C|g>~D7#yQkdS=K~; z!lqc=u54L>jRK~*o4^6t$w%ULX3!t9zCdfuv5ga;wUS zDu~w+>iP=yLF)YUkOR2K>ed=uXTa83-AaRld+8Wx3JRZnqnG{=AneT+4EpL>x}Wn# zJNJj6e!a!RJ+Yf{J7E8R; zZSqph4Vt`(=R3}Hvwc%V_cm+qOY78ERdjE%_WoePR&;Mt)hOY&ev){;iV;S`Ju_U!*UO&B?;j0)X!4DrU^ITxJLQ8Dl&4a21G1G6##Ms5Q8zUqHZUv zaZpjX?#CfigXYGNQ&IA9EU`Yebq$GIVs-ELsj0$jxUbA-ss0bh_7I6C2Ne5B3)SSy|P?rdc;HXK7@2h4ZqZwr1=oCB{lp~V{+z0NS4&_ON~jH z4k8Tj(W)RF>@eJD0Lz({>3&Zf#nj;psyGh0_mt zyS;nxr9#Qb}pflxSdPrByQ(Ye&ropLML$tm(WSv!6kGOci>tT@ygz;y(gO3 z*`?8?_l|{p+0e&6E<31ja1Ja!&v1x`@}E?0;>{#+ z7NdzMz-Qe@MRhyySr@_#g>PCASun#*82OcAPZpF?hm2m06XfEGx^<-Ia%ikmt@)^H zgS9M+Tk2F8B}~G72IHHiJ{A*hp|*&jHa~p%-^OkS5&~`P)_)hf5th~syPX|iH;~?cjNQ7oSo>EW zxf#j>8=Igc>=xNOc5|sl%HtqI;jDjw-N0y+i*Dn%N3IQjWF~V_e`AzH;=P zmt>v0n!2^4?;M+DKIE;V?;M$BM!UArci~N*JHQ;%DSzX}f>OKvuB*OzW8WO}Pp7)y zjYWA>uN-~iRma^psBlq5F`fx0>duHkPq^yn8w=RM2^V#%?pr+ir2mQ(z7-mM(vKsB z9r!kWyYM-j6Z=+;eR}qyQ;sQ=r(!|X$fw!hr+=0AG_IKastTT7RDpBCle2Xg`6QRX zSx<6la(=;+?3R;uNRO6n3pcC)k;X7rlf5)Y7&Kh}GjLW%FCwlaeQ$lq9982D>uN6X z60ggG9e#jFcaf$eqM>$_4D+O9AR2&JAM6rP!&S@=DsOk34)w)GDo=3nmQbfb$wl)A zR}F+>ko0gjWbG4t<7?kTf=|x`Insrvze(0bp~@T&<3Z>RFZ9OHRPH|lTfU=+r>FY! zFdxUKe$@*}oG_=;_MxAPU0xzoDg4ebMpfK&Oz7X0h;ZF+Pt#WwSQ_qf!;D~>33*<(&$<5yV`rR z$YlGRy~J#Xb2(r;eDzIilgAuow#h?QVtWfcW465sY54IDewu^-p5T8GG5GhM1Af(V zgCCrRFL&@~Iru9D|FMIAOz?L)_=D2$c@F+42YW zV;ep@4L{7mU+Ca37yKXxzgX~d9sJR0_=tl)!NCs~yycl}YP(tR(;WP!_k;J{&}{D; za`4|0{106F7YY6>2frW<-{jy|abSE77ks&c|Bm2~a_~2#;Ts+NoeutjH-N8j@cRmW z<5GLy(P{W<2Y;!9FCk`Z-me|}iz|Si@8C~N!%d>>=EE-qf1!h4{a!FWD;L@Lyv2d{{q!~9Cp-941%HBrzbg$t+riIu@Q17Y zT@L;b!G|3Dgf#qk2S3fh=LkN__5VGVO53UyH~-V{gyHYtN7wWw3%0h}yxMGCj{jOL{@h*zK>Y-sl2;gap z)UiyvJssR7Tg~*JB1&M;By@B$?r=jIUL|ju(>>GE7(K0{IqkvV4LpmU*4o)G=;=@C z>BV+7ik^<3rz1G?J2}M_)yavjs7{V^MRoGW6a)Q;rR;>mLO+5+B;4>;9_?^W(72te zKhZIP{pkagBd?>t7%1Bl8U)|`v=$x4La9p+;Dv?|0y{HXx+;;kXmTiA8TJbb<;EFO zh^l}kNe&gs%egMbE_pfECCw%;=em;A_%8m=pcG9$={_-1R%K#^Q(p{2aIR}TGMebn zR~jEJS%|HuS}c7dqaICFSb2uZ==)(u3za{%8*$^zTy;@liG?+i3NbtS%eY;cjM4KB8u zbA|12VHza4SNs~ej0CYV`!num*tny2axNC-fr|x#J#M9`-cw%cP5P}|G&t84X*6O< zZv)T5GdG&NtT2|kO5PRQ$QJw$Z_F!hxW^A^_)udsEIjk_J{p=jeUj$wd0EY!C2q`z zBtEK4wDC>ilp2eY8o#Kq|)cqmh`UHGjdc4>?v?{ya6g%~Jm&P!Wt#7{=h`_&s@By;vG5E)b zdrfTc_G>fs8&*|Pn#H#6?C2x^+={KAda|m@|9gcl*%*8KtjO@x*4n zyIdswREdOBusw=Xf5|O8u$V>|NJdYeu$U>=2SwKeHcFB+kkUTAk7mWbG0Dl|p$(*J z@k*9Nr|s6%4%Zz3E+(69qNRcit%=WKTM2LJzyrNc&2YR^&rIch9VhZwsvrFJB}!^a zW|`2ccq(_EY_EBQ14$o+KSfpUpT#hH_bZkpUe;FI;e|dXFS8~k)@NZ3eg?$qK0^)m zj0qqe=z-#)_hrOONoZv>mHmWN_3V5?zGak6UZXT!-u3)amX6E2o?pt+ zZh6;B)8#cvujO4Y9hcWAjh1)4v|C=IbXnf@(rbB*(qehnOQV(BNHO<%+$nQnWKy`b zy+8fZf^0z|K14|`3|S)Irm}ODq(^Huu4Hb?UfOMaF1oX(YB>5O4kOVoaTtq!i9 zm-2_tG9}W;S*Ao9JIjc^-G8*5odXG!%D=$Q0%E-{XFdu1rYnjBMqE)O@L*RI3EZDz zKmx1eA|MiI?il6jf-@vS1<76(oTGPFF-;52Pyj6)kVy+2Hij04)GY5?h+^zh5^_hA zNny+IQcfMID53pwM;(L1N(%L)%-ee*a@ z3E@_!waA?oDp=&(gGIh6D*Nzu7CGycMZOZm=$^Vh`Yv}}0`p$(FBpSk!y;0Zyc=$) zf%&nl{P!}=J!zUgBhkqibsG1%PS!m-&T6gus51Pe{r7O77{+=nn3vJ{g_j>(@I!o? z7bp(nwf1w>Ci|A*ZjfYTy01^stzA&+E`I~&kP>IrWY8+K&s%+e1Fk`0_=58CGb*(L zLKW-vXU$aTRb}FX-Hud=B8#+tfQ!K-DYv14hU)6^ITgY^3a^w^ve z&5OYIP;A4^fDYhhx7LX9MkB_c)Z%#Jjc|kVlR}HPv`37q%s>y9;>smo4Tm1a*ja!c zeq?6>dN_?UA3a>+ib4+;x}wm-Ij$)5a5}{RJ(Okm8L-=S26|W(oa3pb;=hd^-uAO| z3q8E3vO^Cbg^=hz44jxst%al~IEbsKXiH1k3dt@Xrn> zn5}b2qE$G7j4t2=tV?ji0ZxFa3NQrvoC8|>gP=YaH5_C90RJPXA;^DR97`+&9gKwU z7iw^s;AGnRT-4w~rU+I!R74w3{bqFUCkagqz&i?c6;43jRiB3suG*eu*FO&*T(!ep z|2%wf)h>7abMV3NtKfSrAqshBCk6=NQ5G*~3$-9v?8;1pFeyL?>Gkq&;e*jMiRP&A z#oka(Y2prf#3cEzbAZe5^9+*nNk^Mz?YgIsoG-(p6!^9m0{HhHHu$X^fIoi)@ckY9 zNWq`x;GamtKjz?L{uT%Snfi0PgWvHa@b5ov{i#aB zpX1;!cJLK~kGcLlBlyQ1d~q87SOTz&q>4Y>);P{@LQJxA9C=M z1%I)FU;BJ8J}=I-@p+#E?|Zu1-^amwf4I~TR{XF1bdep`znub5s!Jq5k!_p+);`;NH;Q!>{3)Aq2IQXwS_&23l zt9S6%3H~w%-?ln<-}fGL^PdCnyQKyAKfC^n6#Qup{)sgFV-9|agMU@Y+QHxA;CpF&3S577uvFUKXEzGgV^tdd90z}~ga1hFAL8Jj5&Yv0zBmnktb;$< z!M`l{Lmm9>g1^SWcdiQFcgq8A{yX@If`8lf{}REU>)>0`@QWS%OB@)VQG#FR;0Fo* z>kfWa8vbSn|0f3@6a2dlK3DKtAKn?rlNtKEIdB|x-$ynWEX_hf%P_Pe%yY3ZZ2u@x=?DSsoZ+d79_)PdV6Sqc2V7xW3y`Ztw?4Ub>!+-BGmq&tgJsH zQ0=bOZiE`lOr2wFj8~LB1i64c+|SL6CojOe^~^{t`K#hs^0ZPlpED$R?9gP*yTq%- z@PdKSKqvSCaqWXw(nI1lN3tPeT}Q|Z4_<488UX~oUOdqft7}tj6SZ`gopEAy?W#~j zttVnU-^1x6t~42^;FkKsDD`Ud6h)h!!uDOgbc_t;_GUiinaDaWdo9K|FLglkz1dl8 zw@U;L;)QLu%PYVVIo}}4a7E6iy=JiOZq`xmD+^ z$+>;<_7R*&eX$`{M$WU-7N9~2N9$TST_}+%60QRq>WkvOc82!?VEqd68GjUtas-yt{aS#*WZ=(3Fx zSrne$rW(^1l5c^X5kqXm%gDq-t+7yNoSjG#8{;MOW7D=x1u#5ase}bkY+9$argd8E ztp8Ex*b|D`aYp9=a4!Pomip8 zAUK^E&iJtECw$ml{**%FUet$G*?%@A;v8JjV`U%JqWk*T){lHGrZO6&F+)A}p?&-} zypmn9&~}pS;c)ghJ#7+UO!!Hyb#k)l$Hqr{4JvZk@Vm*AJg@Q=Y04@a6_u1^a@ zNnuvm=%SHs;Djpa%_2~fRYxiveAVi-C#~)w@8^*+2|sX zZZI#gTclH~Y;=K0H<+8)E!1Q;+B@GMHCwdo#c;zYka7Edyrb_%2yppAtBQUsxtmxH=hU0Vqhq0oIEgbCPmFQK@Z_B(~Q8vTD4Et`|Qjg+ZA zlaxj4E*Eiq#}so$XLqTXJT($cP8lFCt+g5eHd`;$5906?sD<+tq71cgu6EJn&m=S3 z9Zfc55~Q8)kCUZ+f3M`_dBTb4#V*H1wSvdUX65H7FtgQDMYF8|S+4vvHg!Ak=yj@9 z;ugT>Q0bYUws*B4DyryMOW1=`_j>FTgIA??d)7lKXjzfTop^@?^J5yTt% zZn*=Lkw8eB#46y9e~GWtYQZji?YYfAYrxkET>VmjuZdQbaW_C3Qzxx zdLetIs8W9|B-!f$d&zZ1 z5IVe)SGB7;1N01rncJ0+ajHgDF}rQ>DznUDBG;@O3j)JY>oyh2En2k)eUSg~$RJ!* zZp6-km?}pfuB!MbNALK|(6(oTwM<3)#Ocf-7Dn}pWN=y|bfhtEIL zNN5p{qcBs2YBBCO-&f}2GRy2Dv!ncFqjfude_6SY!$$eevKts!%NW{hNyvN$S!05H z<9Vxw9@GN@sW7)!IVB2#pgeTmvz}S6bN$Gn(fX-v~ zZ2!u)U#8H7f7s}>J3k>kC|Pg_?euMC{*>v$N3SWcRdGrGjDL)dSxeT}4F1u_M|!Ve zviAxp?Tfuv(ettnD^rlm#scRA3i0v#7?K)`r*z5SwN;4FObc)2B|bVh9fK%3i{$b%8e9xnaInwiTz=#qqL5@@=iCUlwk7j&EP_4}He+2#Hc^6j&nE zfI>P)_@F%8utd{?RwJ;vIr31~WFnW~P~X*T@^Gd-T{RCI|L6Q$M^Bqv3=Z9C z$Kf~cvg62`l~@^XSKqT057^;Z=PC^{ey?7ZC1}R)wUPwPWK{mfjYWeDT*;OD^|uVp zRDbiv{<-GRIvtnSK?TnJ^&p`8_ciE)OdPx?`$a7W0} zI~VmEeO66m4};=KQ6E-56bmbyBBoBkL1|v_$ohp{gB9C z+Ahj@$>zlyme%|N`}N8Az@CvRm}?XOkJWvYWju05s)!n-D|U|pbCGzcNy*ls8c>Wj z1#?X0`zmlGk8K6!4<0!{NSv)lb z9mNAXIhXgZ9OQKKTxz{K>b0?5XKvHNyCl zZv)p;vyasX;Saj4d4Yo%bO*b;<45~uJoHLjzi-6KUaP&Q{V3u4Wbe`c2$=8c+l~3Y zlmpCniL`|0ibWUmJtz(JT?ci#gBps8f8G&-67zjX8mhoS9pj+N1a%}(FyGhS4f9<) zTN*gzF~e#6_ZlMh$eBz)KTc>$Z*wCR(c~`*W62SP3LM_gy!>lW8ML`rDMI!N z+W!kE1rYmjexo>5bYMJrDOhc#x8DZShM-=vh^$>M&Tjxvev+~+7U%bMyKNNumCK9G zp4b`OX5Kx-iz_@ZWVe}f&sMu_0}+3C?9tixC`AYU6b;I2Rwp-s)mD<1TnnXcvW4Dx zud+oJ4~Y6YJQc}o6CEQ@EI-;r6Dj-v2=<%-suKmET&{AHD}$@uwz{$8<-`iQybBL% zdYU=Uryzdbh1qtGgPG;Z)>fk{TU#?!uG@`J0c$2k9YWxwX=PLs{#2?_AzP(KXtkr0 zgviv2_<;^9JX5=iDHZxmW#fw`uXi0c$s^|EkRgfY%EWvm4p?uI#{sbta}OY5?2Kxk z4>;2b2oOW9K=;V7Ep=gPC=!KbjoIeQA!k}eL|a(EGUU*RRg}y5Yh1^vnfORf7+Asl zygMXk(OXDy06*%#BH35$72yO#UP&or5(z17%(+M`&!n1*MbHTa#N&jpROg~}vC2{w zvH7uxU?d2ZSsTpMvAzj=mu!)0$IW`GU4&rAg~l%s9oX$LwYOTZ-=`Ic z)fL?zQu|v!!f$K$XVl*ATg10ibbp4bS85@tW8xO{>D{)rj%ALUGV5;BUV*jzErBEh zH33PDKO@C*%yu=81gUx4hB}H8j3)XOg{KQ+3P$KuOZu-nCy$W+>&}`H9{R64FBviX8bqV9BYgeW zG%i6Ar*R4SXBwA~f2K+Q@W?+a z;jrW#yc$iU3KU@S6dT8CMhn)5&Uo~I5KXaQJ7THq`y}90d%gM>!8Z{WN$eyvj!oww zA}A97qLD3U8!MiXS2=T?9$0y;4*8WcbfB`$Rq8;c4pgppl{!$V1C<+Gr4Cf;K&8S) zXj2C&Er_IYrs77Z)PYKiBB`7y^Es6|P^m*6$^WR!zE)M)Gsw%HU3>m?tVnYE_=%EUw__n>gZFav{5;c6AYGp1(ta%=~i2*^8D*J(T0)a}*=SM&YN zYpRr%w(hM~dA3S@WCG3-LuezrMsY6MIY4pjQ4TJ1v;-*bEFbfM;?{JWc~OARy0uw* zUpIW#twja?Ir7`R!P*CWEC)VY<8US$bi)%G4bM+w&I5gZFpcK4FZS&|ntM6U?uUVr zp3OcfJ4viNrgQ1Bx??&&p_-(TC|P5qLlPpQVTbW7Vz3pNtIEFYIEh=;Q6dF3v+{ho zYoO#Vw$T6s;!Z+P))p|JQy7KVqla*QWs^hwLt2`JXQr~lQlGPh!;91)SRJk8qBbo< z>q)UJwiW*OnRtk}>mA5ch*9XnUqixL3}l=c&a>g}K|iKt3K(LANVvX4V%5q1eAf0!&2gpTfko>$%I&g*!?JzH;QMux zzesI|(v7La4$w8fjFW1|+$0GmCjoE)A91x8dL?S24aiAx`s36zAKfd;K{Xf0B9+R1 z(Bv##E!Gd&v)2Xk)->d;*~lg%+%8LZv0(|BxV^HsYS;O?&O$VAL?e!tl!Z>UWRpjZ zB_lI^5vXoAOz7qqBq`4xn9BpND{>g|9BsWuihxm)t={xfzm!O&JXl+?En&|h&T!u5 z<^>}Lm5LZlesZ&`f_4l}*LAu&NC$O=4sM6)2FC7W~-` zeqkDZzJp)Qf%aFa{Yeh~WWgWp;BQRBU+>`Wa`2A{{&EN3NAR0&*u(xIw$|jY+taSe zX8%zAUS*Wqfb(UPM^*g7_75+4#TT-Fpy9cc**qfh@7O=cFpQ}3h?!dVuzx65fb1XW z;*zi?UQ288QtOv-2pgaFLu*r`bq(s59CeTocyh`;6) z%^E;W=bNj&{Wh7o`an_bT^+drzCho&<)7I!9 z?@G4Jb~X^)xJOF)Xf_ZKN{g`9Fl-$M7*t{{h=>!cJugc|$vh35kb3GZ~=P&@7#ze~@`&@-eFQ#VY+g z5#S^)F>G%9xo~bF=8wP1{1JuJcC&wg#Ihd;eT~X~7=%U=5q6P*^4Gpco=t5m?g%5L>S%`3OSexR zu0K*l3>4vO7V^_b!Qc+|47&<8>ylaRH|Y=Jcc z)9fd3*EwoHR*V>`2Z}OVl$gx?OtkC`*yuRXm$F)HaaM~pVzBF%v;QBRbViHH2)nUe zSJbLrJ4T9Li(d{mG*KsHP>9*$M6KyklfB8cPdP&OPk0p`+P3NCI0t zdpj3Z4DdH&JR1u@_Mqk_e;dcuoDS*_PRi?l$D)Np3Ohs1&y3pIyd%HARKw}5b;%=A z`+%>r=S?++e+LtEe6t(B80hOW``(3b2&E&(lh8&^yy4ZF|3W|ChwLYOg1Pn)A*Fc2?*~cna2%a04moS?5H^JQtONP?w^fKD~sxgSns? zks1{O>`fB1$UYrf&QRa8T0)wc7EF&tn;H z`^OBIzP{^wchle3tuHj@u^0)~S8yK`ty@&sgL4#*nD&clXqxgGLBV-KBRLIAr;wuL zi~;5l6U2|0V%Q8wL)c?2F~5RradGK2QTDRkM>I96+DlD9;=tOpIhJ^#Nw^e#cUqw&(HbL*ZZ2ckAU){eY&3@spjbJ7W~cXw)dGZO%7R^*Huw@Uv}3Ww#{ti zeK}nLYWjB7^o`av6M)$kZWt>~*qd-6rvC2w zo=RrRU)p~H?#BXq^DMA8Z+M;YLv_tNMl>~V9qfjPen$>~9K5R?6NK7S zOwiRX26$oZqa?WT-3uE2!h_o{6hoWr{q`S%iQGEPCh|rOOyocRmC4tiiHyzYoNJkU z6<25GTc}$fb^J8?kQ|$N7xYTcJbY#sMZ0DmzOziQXtF|cuR=aq6~)XwtS43-%VX}X zNGx@{ej&6Mst8F*tYE9@y0?t&N87nu(MXo^K0Q9nFONN33Tu9?SrUmRhvBL;r-gZA zvnHC1GM#7HISgQ4A3sHuks%QO#4rf8T`R_?xBiSIhkq{w0(tWi5<9tfU*X-Oum`pc zOzyO97&G@c3Ia5nsEWe%??CP(vQ|E_uDSF0`Qhms=|uDPvv9{@Vm`jkU2C&zbz8XB zwt-RA8##bB2e94&=;pTPG62>)fDH~{y#W520kFXVY<2+e3m}yNV4`>BBb^T5Z2?@8 z0bp`?*N9}i@O16P z*L5x9U8fghT+4XP=}J7@bzjCCPCp{!TE@#w|60bijQ1+!y@DpQyM5w@!?j{ae*mYB zKsXILr%6_7VDA%V2MhLFQ*6POb6~-C{)GiQ$}QMGvR2y~r|;ao7y58C6SWS8 zHQexh5A_ zuqT(LSXC%OU_7PBhzQXy6@_8hFRw@}f$3$1%WNVfk{9q(IAjh0^22Dazi82V$%_U+ z7v3cj#OrmBYFt2idi@KbWmvfocmLtWvZk`C%rLDdLK_WN_vP{SS6PaM=BLDd2UE?!p8r+swlp7wk2 zadKaU(AU4skYiVAcrJ~P!}X6)Mn6=nZ2KDd!l*1SH(D)USqWRNwv#Db*-df}PAUn* zH&gK?y258d{c{HR6)Si0RFFRyI_f;rVyP-JE0AsRy*>n&c6zBPef1!A$@)$vJ=5nP zAtKpzB{8s&isk(d(@l;(BG2Is!}Xt<3SMy1vf*Q;8^k-yu(l~)bold9$J>gM1kciNeGx})pftnMvq$Kp7e z|As4>lyPOiA57uQ)X3juDWJV;eown1uiy_sn<#JNW$)E~(0(w$ZUu(xU(s6Y_GQcY zvi7mUGp})%7)a2PeehbV@xSpnPSW4_d`|RvX{ucC2u1E2Q{%h?GQTl)`~$T~<8kxY zr>Emv0sYi+^S>`QB52rNHzq&X_k~i5w%MZB_oZ(FlAPOdsPt`24W(NNwJ2#U6QZ={ zUhFYs2H_)i9E6wseB#bY!^kI&m3_wdg@Q6<`o0q`Z6trUckw1=G4WE`j=;MZS)l7= zOZO8r7F}1icRxWe=(@72>m>JLzZ|Pu#OG_RZM07(hW3oq*q`9WsQnK;%l7>nP!Z8! z&yqjoG_D)YCm}(TB_wF#i-G<8t!@^*bNRtwfP&`F&TI0ML0GD0FdA622Orh@gxdoA z^{q(`|8o%jzZ)dc>X5`A!4GYXlY8L*D&x|g$PV!TOO(1v3RPYF8cBCR<>pxbDnLF~ z-{y+Sf^}$bn{Z_J(F?Db(P6c z-~h5@`{CF-ci82d`814fQH1(C1;8_#8YEHL$6oB zHKNGrWKUKYx1}9F=vq&*n+%9)6HdF!jC^dGtlSvd{C$`c7u)7tiSSw-P{fVDu3puLtxB@DhTh860lkeTa ziA2w%E@}ir+Sl`v{qUUF=?xsK&uL%sE7D6O-bMDnDG)cGenot?BaA$C%@2}UD|-Me zk-psA@_I1yTEkb{xDhikSwZ88mxF!d<*)iY8>7FmFj;K7VVhO~UwYwX~ z6S9xISo=}5WEBqOXsK7RGH|fMIZW)G7it&0Tz|8&(C>r>FUhuY#ggni)#0c4pA`uu zo!^PSq%W_Y_?rOzl}n_n z>uS9zXz`5+*5bz;XmPOsHww@XyLvng_^1P1>;R_;aFYQ2u&bGA!0Q~~?;POu0^98I zC|vnVsgTnAMTUIf=mXT`pHT%k4Y3EXj7?Ko;SoU2Viq`CO=Ojr$0_V2Dx^Lb+-O8@ zXq0_m(3ouh26_O?OX6d9=)QRWnRlfTyW>4T?2h*+9g;vlb+yComlUBkdZ$mk~1X3ZF0dJbbP_ z9B3`HK={143!mpWeEuTO5k5b*8$SO8WV&foi*S0b!|B8_g44Iv+%Jao1aSJ!J`*l) z6egb|d_Ho{9^9v0Gv?E-XYihOEzNq`S(@{-Gh;j{^_|n5?R5Bjz3};~pNr3jgU@|N z*zox$K0X%cwC3`!2Y7w4(y0Y_y#fR^`I04qF*oH2SI$2Ry;4(7WiR;dj2K!|yAE-;Gl9WrwBldwYQ2oA!d= zn}pxl<%vn_<99;?IMWEf=cMub8gM-J$m7<$+kDKL4>@k(xNG!V_ZP2OGWxAQ7O#13 z^xI(YRcRbwkU#ouFu1Fj0MgF}rGvrWO5^x~qbv=#t2hy)pG^fAJQD&8o(Vy=?o5bJ zg9_KiqKf0ztXfoY%o?@>6qb+L6bH+{8LrP2me1{4?_KzQoACYDdg6QJ>HK}Vustcg zd~Cnau>H0!Z14CTSk#cPJ$yiZc1U+;=Yi?JNWtOv#{&F*_OAnMJnX*=8(+u)Y<$>l zV6vg$cQD!FS}@t%OVj$u;XgFa_7=ocpP$VYFFUH@f*%_vi&jGjs)M|vrKmXa(pU%) zc&gzZf<~Q`GbGUzPetJ(7oh3(h1pSkvHWb$>-|nOZhXgP$dB+Z(94a36a7w<<^kg| zR<5#zHB;e{PWX^ii!F{=N*gEoC>7oChfOUu@?ki*3znhpSvDlGwK6k(C*r|=UoL4`#1IJev{Oo+DW$IaHd;a+oFkWjwfDf6n|LonC=n$ki=#> z{%%G1^z~qt*zV_Q-)#RIki?&eSDb#BzD30NW}ngz{Qty#wLd5*7ge3#>g zqbGb?T)sJ2sdx52arA_pJuBeo7Ii`FGkO9XU1kONAhQB|kXZpf$gF^)>t50P>6+)7 zcdl7NCVQV>Q__(7Y;Y5k350Z68<~8fVzI4jEr1k$je$eqwj0aYQmmo1 zA9*D&Ixpi7WVkBX0Gitv4Q*o2&=rF0!Tpql41{tREff4h=}_;Hy&i(IWhKWv`whwxFXk_3abb=JjFTo2nfj;a3SfWh1?a} zx=mXMS(J(RIfVpn)CNG>@v6by7J;FOa9cd_ihM(Jg6$&n^-v@SVg%JdUPl$r#5}?_ zL7dmZ&s7wAYz1vkbPhrN%yzG<{p2}N=p(QwDO~ivnKHThq9~b~HSm1J*9HT6>^K|H zlQ}S;Rdoz#Jpt>>*5Qmq~a>KhfHGcNU@q#Uh=-8sa%hR@E2BCCV3v=E2_iwKLDwz z85gLOG8!SEUGZ7kYj_UV36s;lCF-RY zBK#og$K$ID4^oNkA*Sec4EkQ;6FC;m5@;FXYm86lwqizy?stODG;y8Oit3jh03zWs z-%TEStyB;JtInx$c}68(SDXg^@S=E$S3ADp=)ZKWjC%wRR&_=5AdVS?z#V<@aQz@2 zPSY@5&SS)5NZYugo9HQgw;rO~6iUk=;z5G@jC6g2HQwm@v@MdpQZzIf?w*INwEOG* z{ZiWW+)g#1w9)uqc>`IH;rh{f?eGH~+Er9f)K=Fyd|c{2Q^MO;t9z=boI3SEO{jMp zIHDN%uK#{KTz;8XfqQP3v&Ir**=#vU}tQ%ZTf#8s?~u zsFv)Nb*CPu#ykjJ@)EsuQ^z@mkkj-^&l?7aO*wzohF*{di%nF%bCp`fL5AK`3lOz8 ziYEXx<+5?UJL?V4+j&9v2Zdq`p<)+{sc zchOVLOR^*rYdr9gqN&{I=350@G)*XSu8Ysgam}9h&Fie~AahRLn+!FAlQR2&mHGaf zxuC!yCCkx^#kS&8{~mHhpPIw@7>*u@pUD0-{ENa$(68=ZUdwi7yQD|LuZeB!Vu|O& z)3raB`+nWwRj&MXwr;MRk?qcM+*yu0%XMeD?kvxp$@!T33fx(NJ1cZ&@>S+qUw76Q z{u@5%nc76V_#e$*UN3`RBq%AbGNnkRJTIk4rM%%OMJnZuOl5J%uT1C~aT2o)|9IY`Py^FB zY_1ESHo;0kgnK!t`!O`ls}q!T4$r2co^ep(yJ+jR%YnL4P|`WfNkiS^pdNQn%LR24 zP;lyfe+EJCbJ5=Oy{+XfjGh+~lj-Pre@cYsA^)Qp9tu4ttZ%913=b$Xp7M(SoUdi- zD6?w94MJ4hMozOUAn4p?>D&oXRb)T-Pq!V^eyxAo)(Wesy-FI4cye&WZkcaSf{2rU z%2#r%uE`e>Cm~o+Q5)kPQl^~;6(?)_%XSY`-0gQ+`%Q}`@u~P}P;tFwPbzK^vNf#Z z-eaZ*ALK%DtEq!0aA2tadL@H6pFwQD9zHDQ%&LPZsvD)Io3|Wa?4eAhVp6JwImgkr z0Z|rqs~e7Tm782Ss#}M}{B0?7;>ii+vE_rS%qF)y)!GSM*2wcisbfwtZ4geYgl`Lb17WRBD1n7cSE)%>^jIbloN ztT8(+?R!;wA<9a`+VIRU1q&_P*9zD(cWSQT0Rk_xiJ~5kO32dmi*Ja|Z{0`3Q(l?q zgtt~P8c*yXOKzw2(@XVh)=P?`Af#Ocqmk2&fj#(VX_*F?5z8O2P`d*DW&`W{pAU$Z zy%Mhfh57&qb}I*n1&`ba4%G8{;!TCHY-&4Y^iBl0cM0nsNZw~x_bt^C#uS_%+2cIi z<9zppneZR?IDa#J&Q6gcU>Z{Qs6||ovI@_;SlN2#FC{KD3V~T`%+)9H#4cvc$4dOm z@Nh}^mOxH~$;2ydbu{3GK9!8w(SDDiNY+gQ1Lz{7#aK1*I4z=K(jtx*`-wO(RuMLr zz{Jfgm)xS+bTS2=9bxYDqi7TBYZQ!0{)*Kw!}6_W9TCWtEeIJQUxhBrt>)*Ss`zt$RR2T zR_jGb1dQ`?{PbdpXGG66Gh!uc;;Bj1nx2ygajaJT2I@~Bz5vqe$B9uX20{Gt6C67R zG)9TK&ZB5k+J|Ba7E(pZ1bwp{E505b=(-=S-%syMOLm(@$oswIDaC3E^A?vQ6t#?W zzv~v2Pv~v@KyLv($hi=cp2r63*qixWIUPSr^8*4dRSU8k^Ch*|2Iw4344pHa6>?l` zl%v{Lr^V`E9#W@gQx&e4ixI>4K{WA_nzT`+b2F?dYRQ1EA`@en*gJcqCLgfwoeZW* zFz}mtH=)dGj(SHuPJ^tTVdt*s#y`=(I9=VnK+&I36?uj690@c~S0_=7!}8J9VH6o( zV=>)rd~aywO~#+ext4x5>Q=Zp_w1XBX6c+hs2zdY5CgZC_tv?FSof{8qkA;`@7Z1a z?`+*E{x?`Lw-?P(719n>E!^jY9&ubbN42YdKM05fBUI*9ZhYzNRy}bBS3EQjtbkHU zBvW+`0fTJgq^_2IK)s4o74)t}MPT(dVn|nV)LZE(8d-ISkv(2BsoDn9O<~Dp<8T|+ zk{&OY5Qeb{*l_*PHjF^it!3(#9^uIdtn~-W>K-A>FAnSDHJjY4am9v8#5VCJY8P@C zbmP~Dlm1j6Ge9#>6;0KMXt6MLZdU2Xl=CopZTHg3h^IBSGg} zF6G-LSbJT9wbvzBdnE&7kDkr7f7JF0!5%%~2JOnz{(Ylcy?-z%;1R!fj`4_RaDYeL zSBjpjV?oqR&)$cnp$>IW#SUuqc%Yi#B#pKI;xm#i}9l*i#)S&2=aQXKsz~a}iY(r233Eu z_`oxKC|@#=<7797n}r_vWs<+jVSlf+Y_Nt2pt8(uDx*QE3^i4jwXnn5{ED)stC|~L zoie`yZ|+&*MMsr4Ex_Q}u!eQy~rmW^|&HS*Y8npKpyt9O1 zoYl7H4==*paPIuJQ9qVjX5PVP&@GnTg60oKHQ^0U8tUc`Mm8;a`=_^ZSMw%(**lBr zq@?KVd(6r0-P+#>;}oO!kV-8_Cfta)T0R{w6>c~Qw9@{BTOUW(26Osw&8^hI$8%t| zHfT=2^YtzR{+5~maP!#)*xLb47vMSp`UdNtm9NhWeF*YIIOlYU_PLX~mtNh8p3Ze&(Q###49Rp~bf1o@24K{p}o9 z+}_sPwDm^fAr#}CP+h1A$^>Q{$9n^-7_>JJ)ttr{#y5^=NXNVV50|+5NEA6@78pOc z6bU!n=Pp4k>K#|)-^FacUY8(4<^Ggb)^T+qQ`ptx%l(}M@6{dPxH&u{YsyL$^|-7l z&6Zw?Ugb$rj7)jd-Gw(l9{lAJlnQA$nkOmc-bR?#XQLL&>55tTy}TjWSq%scj=gFAEZmu$dnE@iRSik>OnDLhlDsA8!rVR|PQjq}~lVu`INSIMN@S($j1n8JACHDt*h zm5Db=ur0$D#JXIsQL<}jD*O9n8C81&s3Gr`UOta?&*og05{2M+E3vl?I|NS&6~=pP8EcM75WAk@fh!BPy|;S+v>}c?Zlr zh5U<6dVCg-mrOC6TiU1cb5Wz1QdPs!NHldL?{h$83!?q;x{7xMfa56QB_Tk40X%!n zc~?5jYJ+N$F1N~{mPnEvX0<@A)MM@Oa73CN;ra*6obO;P^4r}hrKK7(-(fXBN#mg> zOKpUs9dkup(wxz&vtuzsy@8Sju-rQ2c1<$=NPV@7sPt*s!8xsLG*s^kH@v5lS7Pqv zT4^oD=5kDsNqV_rdktRBmqmc$-Zp#Yb-sY(w~b=lEiiXIT>o34FrIACyh*9`T}&XA zYFJMhAY3?!>3HmElUaN2LdXlrnj^r2&9H~j7RW5wX<&8PZZ zoS~yUT!d-)!KCAFpUDZnFZ4h^#6vESOD>3ba#3+O^|bX}40rOH7Bzv?I}Y86i&LKX zW%FKK+TO^efc`{N1v#$T$rsE<@7i_z4!?G<{*2eA{TZ*_!=Lfm41Y!f`~rVSwVC#3 zyf)LHaX#@^&ze7D4Mr4P+M_A-RiXKEcY|N1C%M-L4IxLw%3i4L2#|%LMbA%Gpm*&x zF>KyEOWS{I?^w?8%k_wtRDtlu{ysqP{`RK$?TlP0GC7kH0bGICxkdpm9{(Dp9@p^c z@vl+JxK?njf2m+*_bX8%7`6F$LcSiS>UBwHRB){?U+C3m;%JSEI!#aLU%`x^EgJ25 zih`KSme;<~{7D*fE?G2x((z3bzNBE4jvv|hX>LBjSm1WimItNM%F($ng(jm-!cW3}WRdw^HVvcH|e#U_3|&&^}>lW!dZD)CTl4n&?e5EmHI#$H;{9PfukZ zs@>eDa+>A0zj?$+Yjdo%)=#au$EtcqzN&ZR_w(C-wsxtS6*5<|`6QM)1&qM}kblXi zHWu3VZYR!i6?N;_+UuPi?7k}dqESPQzio?~eUdPZtyHNwL~X_Ar7r8M&r%3qxKhY; zs^C-S=Ef3Ah}juSotDd`7OiV7Fr5huX}3 z<>-f$2EuaKD@6_rX(YodGG}jJ>BDrV_G{n`tUYc3318>G4wxq(X4+9T!E`jD;C58$_+hvCryk8v&oW7Z zvaD#-;eFIWqXOMXLN$XQ(1LKP)5w6|P8TI8g%Z%t#9W^YBzR0?^#48nkCv_Q@jq$) zer>3~Yd!82|BGpWz}6HE_&iEb+H2y$J|(!5*X(M~$bep2h3fwu^k8{yBHpmh%-J;+ z^lc{eKnNg*@tE;N=z&Pfzn}*&!J5s4on6+Jg@>j+Rf`6dy_=3A2FI5L_yc4zjgw4b zK_uh|h)g)oNP@%KjsFS0UheR<;YIMZ&?T$xZICdEw;uYSz2fV~ z0j*5Dg2p|XSR(!5xM;~s(Ny-yT7_C&DB{7SV{uPOk^LW}D}Q7koyJp_JW`FBxOu@@ zThl!G;R6s)^{SF72EZ!RkdBB1g3wP-`bF4n$783BOV0m7eMjFXD+Ymei#)QXJA8QI zCg=8Ct^^(USU6R9VCJ(J8Kd8he{t=%(wOw8Fo5RVF}17lY8qEcc88!8|70;SmQhcQ z3sm$6%L+C(d?o|_$Ml(9+$A29_^r@StSeX5i6mi$c~DtGv?>JeCEP_dRV}kip7tauhfZDM zL>J>kZ1O}2^Nn0}vp;#B$ecXi*5tuoN%Lpw>2Cfs_3u7^#`T;(F`z*(U0-^4WWtXF z9P_On{v2N%-$?v`Aj7|z5B@iqRBR0%tdG6omp+1l>l>1ILlL&*0glzZ-<{jrQ)f^k z8BW?V41(ekG^SD*#1bpOL~XIuk8`>7Y=DG7Kil&T@58_}uRV8NpyCi))ya(=O3v&2 zZ}dHdWc=@$2OnYrqptyI(xom9Kn8ZgOv_gn)|dlSxc=D>g*9Ar`pm@vHd24pke67Y zabhM3xv{w7^va^2PrP`S&^TKUkKomychjzy2hs zgP9^W9U;tnDz-Ew8rk(#al~NGQ(=Ogqc9f;BY&3*sPrQy*ZX{fr5}05E}RO52sQ>7 zjQ!y;)grBHj*Dh!k^Bo&gs*+Ao21?2rmI)*aVxnmYsv~0r&@7AH*19J-;@W$4%mZ| z;|4gJ1fBt;8-mO_8Q)8my9>$jQI>9Wcy!FKa>!dQoX8S;>DgCV+3v5jl~ki<(Bb?<{4-bU|%QW&+7*mooA zKn>#q#!8ymO6mX}TW{_uq~Q`(mVIJnJat}Q*ocqi>yia*+|mZN#!__KdG8?(Il>%E zO}>sgFvP1Jqr2#(`FFeb6KgS^MhKnn(zoN?;=d^V7u9x^cL|ILTa|5loH`lW>anypBtw&z{8X9>*Ke^Ny9mSq zdgdIeHFBn=3O}G^x16<5E5=y&n;fS%3G_b^^}W&R)*P58Bu0`3PF#JG` z_;p{Q#cSJ}C>g@DT3^z!^_ln;pC@{fn|jFCMqQ(6C6VZ}*JU3d;aQ;L|6s(jbXhRi z3A$ZX&4xW{uov`z!Di5-2D?Ertjwj0Bb(Y1DJWHm)Kl2#pRiDFF)(F}ofm$ki6 z2A#n2UIePC!ZZXy=-sauY9^N8xXM{O&o3Z2l|ySq;G$Df|GYN0rHptkqH+IgW$!)V|sl5|3scz z;pFG2rYnya#*sC3wCsoE!BqOKE_`G1RlUj6^{iRB9?z~;D@BD)J2Mu4Y(sF$DeWgq`WDW6?)LXdirl+k)P%4nEN zAMb%C%|#P^i`3luFqRiZW<(~QJ`ih1ofV2Pqbm4PJ7JX^16< zOEx`iRZk;&w3?=3n>4iL!iG6&AVJiaYfIfUGzR9=W64qJ*vvA2JYx?38Fc{lq1_-o zzDbK3eR5RGh@D1y$M1>sye_0CwL6OoOc|g%&6hqCzvQF4`#2+3I^s@QqR!sOLSfo}s$pE>!oGuYu}t@Wbr*WkVooeL&l9!Hhn>qtC)`tB!W~O$(`12x1P8 zitH7=DNzqZzG}infxf5je_P)pqp8^gg$49@AqN)Tbnp|-^gZz3Uh&hpfP$7#1EZu5 zV~+D#I^GBFOc`0Fswg)OE?)8i20B=KpNr5wI4*Hb>ZjFlLZp;NSDM*qYVtJn&IGof z)kMoqQSFe;@Gg9*MQMvvad=&vn3h>oQIAjgtOJg{*EzN^v@zhs0t7|0!)7hrH+eSx z8MqE;d8$<^+}(gI8*}P~-URDJmLuMpQ^dx0*MJ-yl8FGn_8FUj{t^ z>8fQ^PB_^j=941CSrEc0OE<{NH0oBKHA!A9;^nHfw-Zajsn4THbG@_t$iC7bV}&S) z5q`jT*ow09GB|>^CrPP zv%!wdkG-!S02~R2!|f^ZXm0>USMp3-JoO`Ap25%GEta)p>Z|TXlxOf%pC}9ex$Rmu z$=wp^)StOVIrB9xOR~)o}MlEfpr5=DSau_)>i)9^@0IhiS`yKJ*EXQ8G6w zrS|`N`ZOGb?wIgWke+mza%SN#O%GTBd*{dZBu+U2af-SG!uVtFZRwGx+-Jgvk(I%% zurA6pwJ(%uf1^zMX@0bQ=_Ju3?BH4grGOt$0Z71aHxZ(ALepMpkw5-BMtBXgaW1i6 z%5pccU&?Yiv0utrR))Yd;o-7&0;uR2LrkC~*;)vln4uXn@NTERr$^P{>z;+M2&398 z6GjC`iY?Qm5-vl6)jX5-QWI4Q#W{BVVt`h|4MVLBf97}RO5H>!qqmw7&Ng1l(GZap zW7XhF16H&_`_s0c@{!Rx>q8hs5hanwF8AeKev3jPV~V{3xZcB4|Se3HrrBEkSU zog^Y=jToYEuAhMcEf7l_+k2FK1JM$3w%A>LJFrcM)c>JfLa1qXzDQKnUP#oWZX_zb zPJAZLq*>=97@ddI{z@EQhbUuiCXt9N1}>!31L)k3bV$i-aA2=aJ~~) zQUtI!HB^lqRZk+(W)Ydh19bcmGf~`LG?g2}^*^#1_A)b} zMe4>Jv*mMU87lymXi#qaP!Ilf5Bjrqa1|irAi(RYbJ-w{0+L+}-h19QUyB z1Gs3gwOVyIE~k;M2yvQdpp~fU;`p8I+_2h24NbZTpVwv|+$~>}z%$`$$JYp+{YZds z6@L_^0HY*vlKJNIE8WmdVGnd;Tib&b0~b2W>YtVvuW0|0cY&-m6g8CRwaf2LC5qsL8DRS#XoVKbm}9 zvnmk6oSSlEAdL}VnKV}tF9eY%Bhj*0Vi9}uw4Glheh!T0DKs1B)7G?t6cgz<&4Jl^ z9*j-d3zHwz%NOdaO|s|iwS6t~Y?}QSEn;o1R+~?WG0;23>K5&z&lFYrIp#=eY(8(w zo-5MVL#;iTZBadk8#XdSwz%04vH)G02u#H4Ts3LN0Pzy<@U=(62+Ct^yi}+WS-Ks% zkNFv%$Yq)Qdy85yopf8}*EB4Tqvgw3AO!Mno&0qBGUFF}|NIBNnHjjF zfBUD0oB8ny4wxU$!wG5L*kS@u`VcqhShvhc>@?`ws#5cqzOD@BaVcG9LdgoT!Joec z%wLH;%wGw5l#4}xm@Leu8r}>XB2NyN*MXD|i#i5tQL<+j?YjxBK}34;XEVdfUAmtV zvvH*wjVi9#&Ee_quxx|58*ca=0|8bH23|&?C#b5q4fS$*oC#%8?xkXXQ#rZJ9qC3d z!Txh>9nIK%9L%$D>U>dW`ZJ4$qRw4T`)g72pw;*jm!RQgZTBA49VxyQ2KoQd2`z2o z!IvGgG*GsvX4}0mWg3ubP27l~8wC@Wk>StgTB*%tf38*8?Dpqcbu{E0I=do`<+r4T zSEz;O7C;AcNy|j<^*-+v7?gsSyDIi?yclzIS=;JJ_ZQ32i{0(N*e%EKVz&g&N9r-J z!>Y%k{ZAhrPARYczuSScGCFY0fplQD-+@Eafg3TOHw+49dEHlRmfysIS?&=KIq$d3 zax=1DkcRrPgZhPo8ltuEGoUcCKaV}5?H@<{ll`Bjt#1*+ckCMe-qA$=BD20%g*8pG z*@|&op1yXDH*g$g_(;91z$ljZPqBaK79#EGRgnlu8F+ZBmNqkioW_y-{#j1Qt@Z@yY&n#q<_wm ze~<1*Scqm#S)=2ytSQep8-Bgxwh1b%nldk%%;k;FLc|rvle~<=NKB>4QwGRZKP184 zhugoglMss5*qHV4F`u%JZ>$6?59#vTG1W|cQ8}Fvt`V)80R@WTwl7W&IV_@Yp!@}t ztBp9lXy^#xaJIW-0es`QL=wEZR-CB7$!ZV5tFCp2PA73{7muvO*iNWM30RFu++~$E z&F=!7VSAsOD`&=1n8GXa4u0=ToQ%G_@bGsjc*$5f`39qEaez9RJdLGY@60jD)2zPK z>aV2!DQ7rJUP<&)_Kkv@ZN0~<32j}YDrQl^%7bFZ676IS_=fJ18aJAnk{e5n%VkT5 zYH4zX;Jnn8)zQ@H%tD{BO`Sm~qqA(wU!j=$hNirv-5}J%cGj^})J}!hD3jv3N#g4h z))ANdPK#Qd*6=>aF$NZe?CAZZxoxz8ysv0HM`(xRGNd+}WfyKTJAd<}lG}?Oq(Uz> zhdbM^=7CNg7eCXYcCqKfW`V7nxE2k)9t*uJrA3pjLEA}mp|{k;Pzy41t+h+=qmDr7 zXy_QwIo5{HUxjY>5jO#f+`{gBig1pG>kkoSOt>d%h~`)qn~@RP^c)+8P@6j4sYh96 zK{pB-jSXrGF!$N+D~jrF2i~Sja>V!F&B&wsp1QNxAUCMtR)cgM!=y(9keW0)qpQ3| zz|Y#7>$Rv~t7BS#@n4^M5kx&eBco%U-$ffauIF$r9jOMj%VHP!4)~7Nz-_R0T;d~b z2gd@m77=bOp;v}{izkZShi%|OzI~JdR|_7;PPIgRmhR`M9_T|^&p29-d6OI)5&|M` zvX|3K=TW7gPg-eJYkYB~n~Ec;;TAV6TtDAF-(prfb`kfDjJqISRjrZNQc3pgGci-E zR0o**HjfmI)47-t7OP#l$rh^SgT0S#2i$$2n~F*ee5JsLQM4R6cH4cfchsxHy2ZW@ z=i_b6{C5M@kJCl>iOjg*j{+Fom>nA$HLu%%v?1dnbexydIC3&xC+_or#eB#%y9gTN z-0@ZA0&DTBr*drnzwEsWd{ouB_n(1@1RI@+28~Cov5jq_)P~l!q|!E#0cUhZu}y1Y z(WZoYXiF6#v8aH7B+7Ic#CoY$immOzUex;93Tichj0O}1M8z8hP)rCaph7?(|L^Zv zdnO6m)4u1t@BjV(KcANmnZ5Vg&t7}&b$RaVc^2{G^dX*~7XeomtHHfuK!BdC$`}z< zq7`0a*VuCGsJ-8gzfCug?MHUU7(6q=Y8wU>=(EW&inU5|6U$2P4-*z1?!_UD3(4aofF?FZ80DHW_d4p72+H(+ zQi^@e@U*^C&qr^0lirHnvcp|;+68HlbZ5I=C|5HVJM2PPnz>+7<>lVu&U@^JWyXx& zvfnN&DJG+*igWR+W*haI`WxDoS1j9+dj7)Isl}EX;8|&9*N$y|b9PKK16gaiMbPP+ zzuT)PJh&KJg?@G3V&^S(-eKn*oDYAH)8P*)Hcqr`>$H_)%ZLx>sCXcGyM6yn#zIKa zD&+eMr~S~DGpF`Wt;#ActX{oo5^vgGJ>@~we!6OR%~0|FJQb6T*WMQ{+avp;=Qjc~ zSxRR^Y3e?wJ};!tS&@C;v#T24^L=^wx7cQJ@}pQ#xQBVQW=ml1JQ^$Era?!TCuLhM zXPyY@gzfm4hx-pI)KwbONoiWHfLFrq=2pHjT(&FFI9*fBtESMarr4{d)Ie2@;(r+x z@x3B5PT4x9AUD}%7K*{&;geoJ-!6@4EcMo0!Qdc?P(-}forOYXe3yQfv< zUQc+Jyxi-x`KOkRkSJ>kUb)b;v^tdwg^0+1HjV5@2M*Tu%_6r=~f>UT$;4H#xGy%8@72%aa z3-p?saAk_~N?ESLwnodIb@_2pfAC!4G>gx2vp9KrFdZKN`@$`YnJoI%xu%bPb*@RI zU-p~d!YOm2aXIsMvp0VoM<-;XnJI2Etm*1;w4xuPH-Tg)E?Js@#k!lB-!)`W28kFs z=r{jdd$6Bd8z$M?N9gUYnJQ*~x;1TtwI@=x?$G%iXsXvM(L)zv?*Lze350zt-%nSy z?j=W`+P9#EtZLHs$zb$h`XgBRFfR&Dc$hv27CoH$pem{bURBBZdb4T*>+7|uA}u== z(>QP41D#i7%Pvy3v!0X8atkz{4-xLV)Rdq%85bg?%7g={JUJ3ybFC;cMg^mfP-C$2 zkruuG5o!z;J(4<*7k!-KyvoNZ&YSQ!#d$@K6Yi^!94z&O99}ixT|Xobd)F5&SY!6u z5!noiG*izVZZdlZ2W0m4&moQfzSyMku}33~mjTfP(LD$z)E-^A2<7GE#W6Q#Qx>uWjyM@zVXYpo_ z8M!X9XIFRrQ=y`KUP2#XzUm&M2)?^AWvB!AAa39aRR4Jq{4h196nm(={#ad?gQ&$7 zQ!+JhQwz<^kCYMcwvHW_v9eCB0Q45u+ZBl8eollv3!-Hyvd!x%dGgf5m_8Jaik#tp zEztZrr37nbwdshBG9;Z zP@rj{aD6iW4mxPriP>3+x;2H(%xWh**2gN!-VZcQF__%H;d#)W6G`|#ux~02Cy%&7 zEvwD_OJAJcK$AF67pphELJ>}{pG|?KM&=vVpkKGiB>q3UUH#hlQ~_@Z2SPpJ#1M_U z$pn0^^562Vab zE*Uz}0lZ?<#sEG-$KOp8l_jE$Bz-i51n2(4rRK!#dqX%7-9Q`nhBf}$bmAr zFu4E_Knza|)`bLuL%(T-Rvj`Ih)I1h9s^@^XivDcvrw9>CDf=D-^sD=2pJz;r0$mP zPmw9Y9#z&2kwrnGkYU2u)T4hV!>e!L?`7!?*64m~2iU|(yV0a$3N55OCL?wrN}AIM zsB0E;%;alqTn-X*WwEl&f#lctkVx|EfK!ITxLM`&DhP7EC&Pj`(l!`&GYAI zWcCF$0}8^VJlMc3eLu@*1Ka6D`vpB~UNpWb(%2S^lrhhiM*>lHYqV#C(-LO*X2MZd zBAO5yYiPT=tS+B(brExZMfXu2{lgnwJ;mH-KdZNIMAHttM09AUJvUl53Z6S4(SyL~ z#MvL(Vt2|o_hOy13${J%#-&u2+kN^%SN(_a7yCb0{}iET^x7SY9h`oyzFkRN z8SlXTX)1G*r}D9!Z+x0vFgV;dX_2SgSkvdzQNhfhzF5AHfPnr-<-G2YWe6WE`_tCvw<) zff&-7nK=wz6VEVM$PM#2l9D7W17BA!3Q#TVVT8f)nfU-y2tx21pq-gSYIF2XYX`(S zdQN)kDL4yjrKa=L90fsALs=+Q>6Mg5`l2J_)d!lJf#l<`JPIC*F=??1<{uCbw1gk{ zjIyFMU-Gp<#+QUSz?V$>G8VvJ`})U2JlD+c%lRRt^nr>@aqon?a!ube9LDw z?&d&^qXgI}KrfZz;~Bum9N=mPc)S3c1gJ6O)i-AVZ*+k7IKbH$G?rZ_KrfZzlnmfc z9N^UsaGC(;3DAqgI3@#F;Q+tw0MEx!XjuZlZgf!DCuE?GcTgi7RGF%qFQ~JD+Vh$8 z_kNgV-+P#Y`m~^ug2FtIy*2~&oP+A%K(7oI)B>Qe#*7^*{xACw{%k3okT?w5**r6c z3CMC&mY_2(gj0B%0!0gTaX^O-)$OMVG|H_JT*S)G!Q}C6&b+L>5ct+D`kR zi54{o!CH;0tl_FDi}_Hsr!=tO`~Iw~ zvKM6fEtvHk@zo&p2A+wK{4Sa66J8cZ2`Bx{H9AsNP)3D?Ty)(_ zGy3E)44dJh8w|S^>0DVMBJ_D5ItI%*blhDC{SL)D9KX>Ir2cge>k46;_Y~XBIWhMM z$f;D!b+FBf3M~N4&vX?%h_9u19&xVa$o+O}ssT@i-#MlR<_op~ckjOTMiQ1dIYRq{ zwr}N&BEFtjBB8m`_M2G+e&2b?XkJY;ajip;tEmdsZa+BJ6vIhE7U>i1)k20t0*^~4$<~|aT&fIu@^akX8EJBBVC>hrGe&O zNhmyY{-PILp?H;(6pzb6O0-@=tA%wJbF5p{Z+M3F4bPJzrw;#T#)IHuX$mU#YDV3# z%(xxZum3V5^%VU)6FOxbiTzvzM8h;5RUhfZ=v^3G&ao-lU`@8s3G>TfQGy~R;(L*p zXe^Jf*F+QnJgRs>`G=1k`HoUwy7onSQ}hY}n_|Rl!tQ_s^K_u;FmYeN!=o6P6V=n? z^cX*q`1Bjy9qhu5C4QS3;34NFL#Nk7lfws(>?Pg|qx7g7&!vlPl zD0Jhw)QzVdg1L{{crLZ^Tq=<}T=w3y-&=AgNj>vu7>cnQIhYS8Z$4TYC>$pJqhqL! z`{$=sdn#!UhY7TR!zA9xVFYifUs6fqI82~99465K4kPT@>b^xt`2a`^30->dACar_+`qhRhGZF8@@H}_VBE6%-@ z9n56ftlw(+E6ZL8B%S%jHcE>_4}XB_dwH&xK#I>^Q=RMk|YToA^VFr-w}uoneNAA zrn`1nW!8IUzB?o|RawwA@zMH$nLKnAhP26r2yd_V`=KlOyD-`Lm7rmGOB;0Dc02o6 z*1KlCJao`t+IJ5|^{S{6|K|E%OI!25SEwK-(6rw?mbOxx7YD$%Q7UXdiVHMFNYjPW zX4oua73q6Pr$oZt#X3w_(xStpB^^49Skm?z=F`24b(nB(iw=|S?a*Pwy#VCt76;wp zpj#Y9+#6|36~@ZCnNKWu&+Uh)RLP2TZnY&JMw7V@hLa;6MiVaoP|~HhEoc4^6rTC> z7d*csq)gx1b&|J;bK=k9PaJOC)YE;0@|lb9H60<{ln7tWw4;zr`}ozys!f7HFQv1e%Dy7!?>%bpnwANkwX>2yf;p5rq7&z`}Mvtftz3>IhecoCe< z6Q>ww^V2LioA2%4-`m(j8qNo(f(IYZJouPuf9BxV3;w$feqHbW-p;=}_|r1*BOLrU9Q-wc|DJ>Y zjo_0G{$N4+!}ssD4?n`ee_!wyIQVITzsSKqn}L7E!N1Od@7;-$&$0_0{F#FPf`ea_ zfxpwi|JlL+N$?jr_@e}WV4r>Or5X4s4t|zb|53nCbnu%G0>9e9e=`F=+QG*h{A$71 zI`{_#f0u(lG6SFM;7@SyFMJ00)ee5H;C}|ZG|HLv-7kLsobBQU%!4SL!!&Tw_kSZL zRh-QUR|J(0^6aGluRwvPPy<-l`#Au>_d<%q7ApRN-B}Z8{&y*}&{V<*wuH;Zg(2&L zH8(tThl#vs@`%yz4b zumze=)stl_0?nV{1bdgOE0#Fh3;O>lisjji_w?04R?7=K&1nIrQUP*W$mtXAw3yR} zIPK@tr2g$=z{S9b2XWoz8Cij2FISYkHtX?7qETc8lu{^J zGZ?Pj28;SaW&X0xPSk;H>)Ge)+$GewL<5pvAc-JKVp+&4pm%)IaQ><5X`$khHI?Iqp~ZFi!! z3&PCo`Rd(u{jRU}FXR-jCbQ4%BOWjT2+;#+)RSdG z-OfHRs&Lx(IPLd+Uj6HSJ+$pzk=VzySM(`u4TrBc)c}Oy;V{6QXm-t6-VOTF>)i;z8S84RfAOa zRG_IE1TQ^KW~|82`L2d17%6$tlpCh9OkT#Y;l%8suD4<7CDDh{&Cc-xmo_dr1Eh$(jBou=4t9MQ>hT zJzVjYLl)*)#q_VmTOMxlmNW76MyY?v@77ez0G^%9C0u5?_3s?@?;I%*NiO$pm*r1V z&s_y`6sc_s*LH_WwvZu^gp(29yJ4dB8>3Y>MZH(R!s`*2mBUF#cn_@LJElBL^W(Vv z3kqE*HpK3_FL^w7zaD&5idWKtGY2dVYQb(zYv!6ws^|iNQ zz|&odH_>llbs^e?OVIey@sOhR;bh&hHDTYr?$3S<|G}my4ZLrkt*4{0Ey}8)89$zr z0qalr1VbhB9YZDa9YZDa{c+8AvtO!MR!pI;;by-iT@m;s{uDNvx*HY>e>|f3{fKsn zlpSDxD{C`6*x|u6&y(}6@jSY?}gUz&GA8z#SX_oQYX-*((5= zhV9V|)G`OP%0Zom32xbIf|7>q<_y%04(c8U)uflcDky2#F3Uju%t6g|P+ygXE%#L8 zL$@$_I-lL?*poi`GJro>!kjn$d zI1g>yIS}@>cJBv}@$aK{TcLXdi%N_sO$>%BUb>C%Kf_NRq6WLOKb*K)yCI(B-so=b z)#lGdp%ERr#`HW#aqcl7Pm!23$}^t*0uJP{kuWf#D(Y~+f}KPP{m zaARX87~Trk{PcC3hgBr}McujjMJL%LUGBZ$_TG9WYEj<}ByV~w@jWG5T03#qfl=Se-ucGhtkj2BtR#JXYwE!C z9fcEj9w?sHQ5a30a7owTETo;;xhql!E*p01mcdyQcfD17g-dk8)5SB}+qV2{(8`vH zyFM%~Z>dPkF0KH#wQZSI*6K^`rM|sW+uzj*9f>Vzj1*oiiKFRt#MtE$D7)$^2 zZLND}v$wT7n*7G~yf#)l`d7v(ni-BvQ^9aNzo_mUjo6-Ub{m*hG?I8&gqDvX&cyvI z`Fol_Sf6%plv4+x0aJgYa-t|>3%v6P2TqJN-OSrLx%uXFUBCw6u{f(Ou z$2PC!G#UCPDV+|Iwu!2J?M#K(Q{=5m%EzH`bP+1eNu0%;&LEr^T_hXAjcS+6^fVss z4xeinrqtdSClrO-w(LCydh1ih(i4TIH_Qu^|C=GL#NmoI)%_FxL?E5Y2wD4 z>{EcVvQjaEHMUR2W9RzTo8r7;0>nmPesY*F0!^|~^z}>~X=gAOMfU9L9(u__^$VYxm?+o8c5=ur?WY8j%@Kxbj9(|sk;zPM52L}C0 zor!BerI~KJkg7iK3#hB_uFC z5Z?2GJVG1a98xzXmdrh#I?hX;zD&Dt6toXT=i2jasULkFExl3xdSWhb=o`_rpfa&%A(Od;G~ELn8C`n140^ch$I;!n?+eOc(? z#X3YEZqcFg;SL>!Kg{nnp@($qn&?Bibxq|%x^>O)hcJ=h_q3?@b-N%&oPqGn_}xw}z%E)HmKX3Uw|ADAeNz2WKri z4tkL$WyfRy^Bv&H4luhHzybh`lP19H4;bQpLY&>4{Daxy}!2~{c79}r%nWB92nkD9F`ov zuW{edKx2aC?nFd31jyQvm@ectU1)B4p?-?lroP}KCUiyOJ3)9toE`iPkfy&tDwyEQ zE8-tA)jj)g*vW`z*N$GwYP@<>WUzOV5!+eNORMxi5U%8agJ( z!wbe4oMAMqKBFQ|EgHepe z9^QwaXl%rHP$mna@-@B1x13iy;t^iI4e!h8Lq@*Z^&85}f#dAth*MGH+P{pvh=Ba% z&_!!>h+YJfHLg9?dBM7g>t>bwGQY_#aoDy?|9l6uLx%|$W!X)_^xDm$`F6?HTf5A? zQ_AMbMT>O^Eo#wW(xMI>auU9BCe5>p0YC`qKGElpJlqRCk`Y0|-lZ#GpcolS5TA|FN1jSRb@020KXvv1~HyRkEK;uLFDrf;( zZVCywz2~JU`-REJU3YxipHw|_M`~YQ=q_ABR)k44ihp$PkZIT221@YVRl(4rHywbx zkpFY&E^}rMEt3Cp=q__+4lR=ZbLg&I2PXgL&|N`yAqVKtUHR@p4$z^y3UuLLWas1! zU4aVe<@$wPpJG%@N|V!jFN%O*Xz5~pv5@dvxs*8-9Bu}Q&=WQtqEFa#sC>ev!|*4Z zPvZXils4XiD+|Gudxb0iB5IPzKGUDZR^^)wTb<7VY&Db}kIOzk!kFd1Z3DgBy>)-@ z!h!gOt!o6si^O*m;m-`yP4b<#_4@~OR~5X@&+DF&9xGA7raf7hZx=J2xUwL8#?>&+ z^X&=`zcZ%8IM24L3gUo{fpMPduAJs>F6^Jv-2IW_XrfvOrUixF&<=57Bb>r+n{$S` zQzH!r_M!NsCS7&a^HjP)a&)5!aPn8vpf6c{I2xWYh6WYcOVbTH#_mPnFXbnQoH^zW|Jc?EE6x`?xwX}8wo80gZvj){&Ex$Axld7BDH4JF6CDyj)jWFc$ zWMm4z5iQ#gXkM)X2?J#n@ZV3^Z3Ks--M9PfRp;2Z$7=oVqcOS518k1>(J+Ut?y74Y zk7NJUn=$lzuYwEhjFxoyr6z0B%J5S!KpkcvOoYo#G31c zdf>^yLPm#7&~lgvQUDSn;Li|840=Ys8clvBb}{Jx-}>n%gadppM$0gQu7M&&Ye(P4 z_Ye;3DU3#qJvr0PJp|wXX4aSemkdPK*S5{6t0JlCoSAw4|A;iU5xjg$J!#ZmBa(96 zqJhtk3p_mH5=(-5AHVh#NculymMZJ0-_$jQ{^?_I0?kOYOrJ63nK2zoK{WAw34hpq z&@E$vfs#u^t2@HPGt9XZbzkg>COr8vD@h1*Cl3*&Zbw@3Jc4r@VrFvpP@Thi!*-WF z4`=l}Odq1i<{cwPJ%5 zSwXgf%j6TX9=7YV4-ovZB6li)yGaZUidUoyb z#q`evM4;q|PaB>2qPD8yDJ*A67~CggzJt-S4+D*#=NVZMDGo+_8>7hCjYm@kixF=R zmGs~hhAuFskj0Q51(5CI7w(3>{5{LYeyO$(aG}_y(4jX`;UNTED1xBO0M=gFi$jmH z|ABoBk9PKZjIue?szOU`vqFel2Dn`ip{2H2A@nGlA@uYPS7^HxbY+DfxWXV;d7(#b z_eAK?9#@#}3T^j9=+XVIu)r1C?g{^+u57r<{Erg+g*i_a>~m>**KHCJ(*6cvW{m~> z*zL(bch3xl9*`w87=1vN&|u{QvV;bQKfwB_3XvL{#iZ2Ob4Y(pYV0|as+1c0ZulU3 zEYCfb=N`)&{(xkKSlRpazav1T=pU{<%~y#2<%#}%0-uvz3Zv^B^iO-$DB`y`KoLL3 zQd)Kv6tQcgO^WOx8Nh4@c#H!)fc0_S z@P9bKHV3#$fIk+X=Yuyt1NaLEc!vZ0yZ|Et^nCDYGk`yIfYTh{K`eI5ej-562k+|{ zz)=n`;sAY8z}+gqPA#!Sc3uWB-~bC9;HnR(ahw1>AH2?w(qFyxW&7$q9Qf+H1^Dj* z^nCE1$N>J`0Y2*h-%%SY0W`7dwhYv-9MlpAwLwtd6_muP=^3cY9n@S0b+@3d6qLlO zSO)4G2Q|S#eOXPr3MhQ=S~tP;{ONVm5N6hAKmKRVc5R?B{owQP6OCZDK=Urn`dQ!v zaVQ^sDQw42K#DuWgq-mcSdKsrXN|LE*ZYD_z13lmSSyk5TCe8E$G(IzgEt>J}w%BWlOUrS*3r7=YbDYT08Osuus~a7|AjNq_ zQd0~ibT&bGCZSTU!?uK&6b{TyK{;&8Mly^ywC4|;%yl-wWd5JAc%Y$>=`Q(Q+??wU z%aMagyCx31i*z2;L{HXb*$wR#0csp79xwe`a=IT6tq)Z98lJ*EZk*2LN*G41w1K0- zGI9EV6|IqnONUMyHy;e|!GYgxz8;AmnFNR8?lB^;neoebF3AtwtbNWYO@4q_W2&@4 zZZXx#o`PPAh^}81nlN8|e5kz)m2forN^AtXbes_$j1f_H8+cUTve=He(5s~~!;-CP zWO_exk8bMyaz%l(U@T}i`)AaLq@&js1e(5MJ!vLW?GrFnCT5n?2kAAcfx^`p_h8C) zP$rcO!k0Et&=2#hWVS!37Io^8RkQ`H2Vn#fUY|LSGAloqR4b`AbgL8 z9FuG@Rv9(Ks>JC_RfdzpbL8=+7s@-jK}QQc>-nSmR-BtG`a7je6}PEg23X&ok-MYu z&7EiSUi4d+ipt%M^5{`>e1?Zy(4n5@O;a^U?oA<|-&z|j*$ms|LMuy?B-g}V7>@%l z@5g_c=hF0~;;|fiZ#vqUm;QsLQ>Zp;PRVPpg`<3=+c|c_!u6XUOW*MjqSF5gP0NxAKD!m&tFPt&6`C6cTxLZv@N*6#U5wks()x`f7#r?L9JQm7(Q zoYSM7hPLoAmwQwW;P@C6&kjVA-}Ms!<1`*R-g~GJ@9$#QgE{GX6eFOJ5u_h4^)F5B zogF%LKeTpGl+7*0fd!@B$d&WxU_H96bM*%LtH8=})PxH%Q)IDHcsA}N5*9nKclp=> z6S!HaLSavOgmnrhu0w_1YQL-Fd)QbkPuSA%o;S`-`rn}ovruWS))76XH1JofLREj@ zvJ)0?M4eH6^55H$(cpD4TJCrj^*T40Q#|1HVj_=d7+f z5;dU(*Xi0x{R`vI!mn;7Iin1R@dI3J(*y~ht8gJKsN#rkHPH=YW$)TU(@!K$DG>&dg|}dTqwjY ztU)h`?+i4(!R)cu_svIwlSa@|3g5$RwYsk>CM)WT+^Z+Lt#1)}4nv5bmH8 z{(t>89dn9}uy$D(;tjH7)HE(*%n?$nG(OdXl+XI;`i_hciR_~H(>%;)FtMNo{W1r5 znC=5G3U9(Fr1`9mcBKLLA}9iQxC49$gWj^g3ee-T)@1ba~{fF7T9K?d-<4)78O_^JSx1898K z$PCmM9aOo4Y8BK9L5a`$R0e9OgZeiIby_D-DM5+PdTVF8SGGK7z48GEKK6a|=F6T0 z3O=hEoYWawzaJc4o;4nRs{Y8-zF_u?D+Xs(bw$7DQ(xSbspk$?&!1d9llD_j@fWO~ zJk_)R`GNeJ)pxj}VpMfyxw_8J)ODV#tH#w;pt?r6x>7Gu*8|kmkKb$DKG~SPSD-MS zv)V;EOwos(IsoXjHemOekTcHY|>^0kJ_R^n&2vZAbg*5hJ? z3PbT#jhoqLu`gD;R(b=-P7npZ2u)64!4&w;v3Lc_MS_Ik=fIKSY&JGd#OAljSNU43 zWPd2WvT+kmC>RqXB~MqNa^uEWjYA1Ex3~Fv97u!yPcZ=?asZ4loH5e=vN3@&YLva*GlI z6~Gn8!@j-d@4NX2!<~!oo_Zox!8%40KjuqusiSe7>2Q-qI`l~}bSs&pM8*pE&ra5z z=1;wR!J7)Vd0bll2BBXYX>1u3ljiguV1>2o=Il?cx!@h*yFNK>c&hz^-4y24U2DSr z>2p*(J=Oh99(~6ieMgUe*!Sp&2U6=U{E$btc2zM5X@1XZzLEZ@uSaQB@ z9_Ys}re3R_VvaQW)j3-EJpJk%-8|=nCa4}EkDC#mf6YtNR>JbYk1|L^8!ELsQ!&a$?|ld4ABgXiUQmvMK{0R4 z^xx*{qQ5|)i`*pIg?|rQ&w7>OO$vli*{VSDB=m7^<+)YW8ZBAL^kdUdEdFe??H z_Lm=L6%l>@RF2{+a=M2JpGVl2>Mfx_(^|}}gkiId>=7@_5n1h`@u!)qt>-4Ahhc>? z3tEOBN9j{tc^gU}%w2Zi3f+&cbl?gVkgjy7&c96;`L~f%0TbBs(^_Nkk6_FTr+tSL zxBfEqRZnm=OmdHout&XDxm$VeR^AA!Pn$5XH$P2g(?HY7OhGo=V86b5T6O*+&6@m0 z?9_;ry+Xbx#ig{K>kvE3Fc;WYH?*^w)RE$aXq0(% z)s@#R)?vbREjmoPu0w|r*GXq)x7c)`TQl?Q7Ml*PWBo%7mg{@lm9q1;jJ}R9tfjg9)-zZ z^2u1(L}!aSn++gLFh;KpBWXh)fBRp2f4^#>?fNuTEByCg{2mZ4o(hAE)&!cz(V<Ho*Gqg0-b3NwOX#^^i_2F*5D@D+?Nwx?5oih54|E~}#5fg1^OC!DynkZwn=k(2*H z_vSYj$1)Ob(~919ZwHtGv60}-oLir!E1@Gro_{{?C^TZc(z zf#wv~vBZpmNTR$PQvt80wy=1kKiZud*)Z5g1E1G?8ztEDuvxlRJjk_+WOBZ>q8!lH zLtmfotSs_wWb| z28GGISz9f{%|*|Mz|DQ#9+t>f_9Xj*xr&v&5onIsW7`AGcnVr(#RTcw3KV)2MM}h2{M|s) zKCGXTXYjt5XtCH~T~i$w;5WW55`SH$l1Vp%?`H<5+$^LXth*U}KQlP<=B&247cHG| z)8e+d-(6aHQ%l?2*wW}t9c^>RF7@9OVWUE%Z22?=Q(8$_CE`t0wRC4+vQSlSnROgq zl08+esgGv1=r2^ZL*8)ezd-U{H4J{CsR7HLzGl<_{%Bt?_x85smoM|*fhje@?y+~o zhD;tZteab<7%3%V1%Ax2cdOa_CgN7on z@&*k>-h>-86nRBAsJmljYwO`pR;xRTv+9R-E#(ypeZdtkKe?u z7R>#l=x^?Q?A47BDQ!Eu5Q*Q1t#R56{OS7c>pc2L7p?3*zq>NaU-Z5lcjW?<-IwdG z%+?pZFX*n!$QQjY-(8t?FM3~ryE408^uFQfM=|Vm*2mSM(~qW0x@QmMS4EcFt*ab8 z=KF({)+xaW)+xbB)+xaeH>9>tSetrt!po`GCTvQrpU~wx&ic=R<~h*35t;t`OU&(~ zDbA|)szEcehqi9`BCr>{dx2-LlFn$_;p^cZsHcZvX>!jw~ zlUmFdUhI-u^?1EnrtY=(d3)9qfG zAJO-R-1kS__gA{_AASS3szX}3b=6TV-6_?TTDp^}C$MyxHtsQ|6!$cxxHmIzc&0CY z2OI6?lC=W|5V-AyW8=FjN?u_VNA|oxfD^LF45|A}zxl*6kYt)Yki3!Qba!}VurMLqql!YVO;5(0Pk+QIXkvSwEJc6nSyJ)5Q zzSQ}oJ(%nHq`gc+_B0RkI|>yTN3ap1ykh~|!tGdXxC%BeD2Ju_A1Q%5s+8gigqlF} zbtlS`rmC_cQ9eNmmPl=RjaedPf@!@fSUyR=;mJ(S`7XyT^P`mT_gDeb@e;ytyz|W% z7fnnpk0xe>BZ;YEEDK|an&MCbj_N#2qh2&_zF{&SJOO=3^V@JPcH=v!8=w$`7VvXVtLY{7j9>f0_}**Rwg z+n@+nrFa6d<~IWie)%apbzyo?qs3S4dF7;6Mz)}Ut6die5uhH^N;rA>U>r)rfe21Q z4|!qIZ*t+i%Wksx;eiD)DzOEK@wvjzG1bzJgW=Z4>=DvDJY+43?`-Wl(YJH$UhO{c zOoz3t$y{9XSMH{J{ehcEkpXeU|I%L(xU;1qXH96H7c{x|x$xTev?WP7)rtAQ8(7e! zpmA~1A>S6^QWHW+L6iRT*d_l}b%t!toc|p5@GMMHVj0n=okJvy>5=5{F*50iF(6bM zx3;Q-WZ@%z@|d)kg3pi)aql;eG^=m2~NFT*Z zG7au2Qp{G{-0B|D)a=xIeVvNC)nTVy;<}~^(6>#@$_~{{x7XP@Rkpy>ulUGcY&P~KD1Tc}>&ucpx#_t>|n zphxfJn^ggOq7))+GN%fOqc>>@R@p7Hn5#))_#8S%4HlEEZ_rriv!aZ>zLZ3=5v<5G zyjTw~@a0`ns1NKfx}_Z8IAQn{)tFqH=>d_iIrAOU( zDg6Fqib#^EA((1+KTwCYTPgYZ?z8~mF{SFmYKh%hiCC6$H_%jp1x-T2D4VRBHyda! zlVniarbkX`A$bp*zL>+v6!=OomAzQ{CA-2bPr{_mZHO!}*EW$oy zuE}r+rcP!sl1Y=rYt>ZK7o#xyQ0EPti=s^S^#Gx+;liK#=$Gy6#GY?4+9&S3VrbA^ zW?8?RcL#$(Yc+H;cu2BMp~g=iu49AUbbX>Bmn^0^q3*J|VfS5+Y6Qh7Yh=~glJ;h3 zcr^$*inCQC!CYAP6|$YI29$MQu~2hXr`iWZ4|eDjBRkR8JV)p~o z?p-(75re9p)Te1<5bw6i(Tz>E8HI%&?Pk0u88%BnIZ5z@GXwJ#Uyw9Wx6_ELoezJujazc$aFYE^F?1MJ`a*O->@7Cf7knm_LLXE=+Hu)|1?AG*OL`j|Xnc&`ANeZ&GxIpX5k!v6UFBG<4&k5G z@5P1<99d3B8x6PsqRlPW9M#nc%c@A7VL8RV$^~Ix*3d2}>T0eeV;2gAHBcnnKhKWL zM7caM=>7T0zL_NTh^7-%OL$KoDt)C0nCl;Z=lu_R^TzmbCf7z4Y`B02`euu!f*ZWJ zmPFv;eEdkhkw4N+o?mH{ZT4JucG6$QV7@|UPbmsA9UeuMP@S>Aw5_;u*l@z;uX9sn zjqWYCfo=gIcj>l2H$9Sw0r>Eek4zir*?S4eQjY}Yk7nJbC#Q!)au@*{LTH5No};Oc zf3Bx%ft#V1%*gpVquMnCIYIrofd_@=3nkBaF-qyF9lR*BK4mDSchqT}4S(~x%S00j z?zaL>EvDKeWrxq3#A0T=SZWqaLp|?0FR9L>v+^+k zFwneM1a(ipMf3)`eNFHsJfKLL3c}RMxsnnTYp~=J)?o!`ZYb~u=Iw{PZ%)tSW7G+U z&SS!-wx|-h6DkvFnnm~IhlC3%iV~>Dp!Q)a^fA*)||E((2G73yG`FxFEiDgqa0b(qzWY_S0QZ#BegGt zG4;<+=pMwQ(72R}-6>?4yC}mTpO{>ksG1P=1Q>N)Tk9bLjJP3h-S-JCR9~*zcu#p) zWV{flJ_%-OF>3PCP~yC_6f;AWfOVV{v-^{!m>Q&*8PxWeG0`T?w1=R8PMT>?OEW=w zK~l_WmdCL6yFDr9QKqvPp(VUYrnv>rFpJ_Zr)8LJPKH?(2w9407TZO0i&QQwBP zO{-36Vc|6y+fH{kImwPlH6FX%qgr8QRV$c+^i(cdqZ%?fT{;@9X)kte`Y@*TJ@yUgfV;{xBqx^JOXumO?3<0VQLqoh={r3V z!+7Kvy5D~PA?W^ivYzu;nkI>ML6b~H7)e~`dLHU1EI5!hp+61gF41r05>U8D#~m@E zlHtc?P#ag3$NBB`KRfjGcUnQ;>(lSl>x|~;1<@tu8d$j`S7QU&9PlzA@`j(3d7CaP zPl9<%L0KJFs$BGhVs31Kh;*~IV8gVKaAS6w2C?y4l%nc30(|L0bt<$)eJ>^Oc#-LF zpy^Xun~L#lLq6E0^ip>zNvkF+6mM`31?ku7qe?wqS61tC zHoP3dMahA@P`Pi_2)T6z?tf7|bLap}g3*PMdX3i(KO8>Txa>7tRUXcm+E=(w>KfxQ zr=$FV`OBcA;l?H-k*0^yyocRIqkIbR>9ZO<>(> zlNd%Zv?Mp;nEWWNwCK>2Jo7RB)}>q1zBv!+NbnXhA-1a|^T~FK6Ua>_j%%^Qo{h4& zT#lZoIpcvc2WwBj7t~`W-^!O9Tv27!tO$c59Qs->bGeo~H?-6Rpv6^gN3R-1#B`M_ z^&Et0l$LvqJ>r@sJgU~F`=l$bkIc|#zGWVFEAZN-l?w&vBv&`TnoRifzVue^Mtt_A zlKAXP*&3i{#ETj_1RulyXxPEWbeR*&?HeZ$?Uf?%^9&vdG~I1f!y!O@g}vMd*#O;x z!c;Bcg90%$34ai31S4vBi?Vn+hSt72ExO|n;SRo7-y|IXX3aajOl)3D=s_V;@x7pZ zhe2iBtkF-ov_SSjbY-Es(rFl+;Y^~x`as79>QL8~+BF*#FaonD;z#Pm95sEQ{7+(G zrYq0rIM0p}?dg46mM8^SL{5pIqqUG@mTCgX7Po}{K+`$uH5#ConAA_lQJ;sb1M{zT zg19FoLJA z<^r35IHv~y9w6U8+4;pkcvvKCSR|+OVpq`QL~e&!4o{Z_X{^EdJHKjWLDk$Rkn}mRF-=mIebH{tRU|8Ge-@ze;#Jw+&QG{~DTO zUizy+;^g&47+?<&*xioH{p@|j36){&dJdd+2)+@D*LZgQ49VD1V?FZ$0}h)}7*3uK zbG1!z?@+TbH*GYvO^EXy*87BW%CJ^Fv84Z7!hBPS*Rl|Vhl86FJ%m_=0oUNW6-yEs zGMOvP$LQfi%Ehk;MKe+%d_&N5O~LMJxEOs}0T&UUW&7HpsYNteURY?QhR`~#6h5j) zShB-zvsnJ>RpD8Rs&j&}t&gXh;u}0wXARTH*wbW&Jzb1?O1J^FEPV+A4-efL zZtW_pJ4(tZmPsqto@<7gzpb2+KD2_sg0SgaJUy23%~=DPPeO|Hx5p;as|_mP3)Gt; zQr2Q7g7qHjqtwcsCR7VRUk=u|$EcYie>l92Q+3AvO5RlfmryZO)xfFGG>+w;ld_B5Sv^XKFz6r2^s|nj;W!Zvqn>9E3vG<}N-v>0hi_ zqml3^rbKE(u%Z<7Oa4MbLk8efvdXdx*JlPZiZjxTaqh3TwIa>rQgNR|qfStu#YsiM zuHlZvW#-pEN3$&TuItV9A^Yc#r_Jp3cbb{~S`L`mzqt|v`tC1+#BHS}vd_%`MjhZq z4)D!~0BjJTx3%cx4B%%S;OP!bGrSMR&SUi~Qtc%o|TJq?;T6$}51 z8L0IR>Mag5@DHl)Fsd_efd?{B_c^F19MmrbHAGNF)b)!D)I0}un}eDnsChu)EpU)2 z(Ao8Cyj0WpV}O6EjFh5-9U48B{RWQCWneQFphg8KUvylb%fJg1N^B8$MdBxwV-sgr zjZIuNK{G^;%I#A`R`Zy1&}y3WU0%*Ln8g!>64S8wpNYkPVzFoKH?%|xgx3Zd{9}nT zgTg1mt(=!e!APZ?Ah>{mBZ&*kxjL?F%v)-6E*Jy(D#uB20IpcxvIYsVS@ zh{kuR@2wfOMcV88bu7Lr_~5LWe9py{vZ`V`Pr6mY6Q8I*}lkpuS1BLP@_;#|3J$lBU+$c7EBqYmQG-6R-XWORbo-A3Yf_NvI2~w9p z(ye*a1fqLa4iC3dkS5I`vqr0-X1uh0r{c)*z2Zx(B{)!oWAUwo1ODm&{dX^tv0FA~ zu#wvdsvX}dZDYj^R+B5xw3|kT;yt0{3~AjOc+boyIjQ0#klbQq!Vz%^spHd0LD0Fu z$eWJ98vl?Xa<#pf2dB&B>>gvzN49j`uBuG`=GF|U?jrH^{UG|}xk*KYFYtI7CetV* z_LJ`PsL^De2^Hmb-e?7o(Ohgp*u$9rQ;Kp$=7l=aYB9#Y!!AuXhIL5hn5Dx!>rmj~ z&_HVpignaM%Qnik(ciY_*ZRJPAg{(R%_1BnE2Eh*+7od8Y`u@wLWp(YeB15DW%AZB7Itv zOj45wUSJa(yumLRUU-879znt9?0^!>2_fnRQqNaOMsML^$%ji-2}f~tfhOB;#V#A^ z_-gcCIFH5LA&MDOxsFtVSeG=JHI8hTW}q11&Kqy{x@(j-pvx)HRMqfhlzX1)!M7p( zew@cyF^E_mmFcr82@7cDr5mX5q@|vX>B*x#8&js)huWGZ6^+u!zt*+|v*1^+>oW^F zHcx$xS@4APjTSQt(%+$BG?xD9x)FIfvxRd`n@C|l6Vst&t|D>a$H<(Gjgh$!dZWY- z;lh86H(Q#vO-s|ZrRrqqhm5dgY(QzkwuAml6SgWx*c|rZyj*mx(*Ul*%`HRQghBdf z+bFI=u zC*WR(zT#c-wS?p5U?*Hh0&N5_7f=KE;9V(-Xzh;m75OL@EWeUrPtGEkn8OC8UZ!@-j6izv0CFk$H{ag}LTo131M zW|@6;B*+vlEz}2D_z(0fo|%c!h9!BLO%J9cLD#CqX$?;|kZ#VgMkss=Sh}EV4(*nH zp@AdedRf})$i6;vUzYAWeW_KzY=#U{f!2!Ft(KmVU9PM23UKT^%GHD**GBr@My&v0 ziTbvR^w~f+Wtm*-hSEr%S(so`iXGhp8#rR4kylOha#xys2z?d$It8INFl5#iI!J9H z#GR&}HAfC}u0_ZtSed>&kp7`rVLeH>oa!;V?W#&x0GdWqLPYYq@ z)0~XrB#?Yc(}`iNk#)(C8~m8&7NpM$umh*F)Ee}d;X|=MrqVo4s9%r#e7C z;Yhv3c*H<RRA^OW-8MXQ!~+J$@ObK+O@kl^NLXa(?}7mjbM3B zVYIzbID)Rx_FL+7FNh?Xg+r7}))umFLo~jZw>bS+gyy*0HyOdT4CzcS-iZ?-b>_!|oX4;e@88Kg^`s@F1qidi=3E(i8j{AET4NO7sE z=jj#n-O#-aMj$C*Mhgo9(J^vrQEu!nAgGd#zDndaIULu^K1RLI zk;_~vM}f;ygEWGra@Rr@qXJuJ1j%ChmmLVqA89kpeZFVTdqCd}dPR?qyz2VF9!_I5 zY@V9UlRc!lR83Owk-sw7T`fX*{bs$J30()^Tw%wG-Iy@Ys(BvAz=AJ-2nvCPkM9uv z*dhFJkW>NTTl-{8Qam|kuUe;7ZeI8xO4S#JCeUr8?DWGhu6V); zs+tGXA~u-GP^4C4TOu-q@Txjf>%Ws4#qxCM8mJ9+Wd3mMx|^H*)&srJMlp%m`xrp4 z2Jr^mGSbP>W}qF>D5xp)T8~}B@)#V3368?-vLeCjZLy$XFwPIT?H0n4#Lxp5>vRZ#cX`QDSxm7QVuc7~pL9DV%_ip9_TX{Xb zGE=HBgIBL$bGANajLz?eY5(0vfX#z zp0;2)ixbf&%;MEx5b)vD+&$I#4~dKvc=o69x{qCHv3Il@X|*}y>vclX)(n8c9*~-; z@)>q5yS%eR5xf#|AyORwn=jumyrrzD5>O}0E$n%L7sbjhn z9|7F0l`b1qV8K@m_h73Lan<(xsp{WbRhhJL_GNUuFu0a>xklPr%zLb(nOyco9QB35 z3D&{DsT(9~sart*j){M%*#}_hs_FXir#TrPLWLlfdrE(gB`PiBgP8cgpL;<+vkGJab zOdFy;9IR=OB6?4?LR!1V>^6MB+%FZ%Ov8iL8AmLI^+xLIcuI9Tn8v z;PZu*H0f{EZL@Gi<6XSn@%icpA673xc31`G9U@y-gI(TaBuw38=B{~G=sYhn=B_D{ z2nP0B>s5IZj)3y{OER68?lYOZtmC>?LeDeynST}Q8)E6oa(1t4ew#Kv&6;Dzrv)4^ zD~)5T{Sp|t$VZ5DK7*7T6A6tA|#hzXy>zq*HN2QU*{rLZmFa&o| zkj{GIh(9pK2MJ z`e;PUGKNw)x%RZO0sx)3b7sK>+25%foW&d2d2kBT85bNT@x`rdb@SH8a0g7EXD$ky^L}YS(^hzs@E{ zYT4*4Uya&lbR31fNXPMrDmgmIK^!2xMiZ^p9F&P2VF2%}@<7u|^r*B$k|#$AIGoxh z+Z2m|T^QX#W*g$B@F}{F@pV}7btxgZ3KZ5IB7gCd@(F*|rq%GyKeISUaYDeGaPo+T zgKRI6<+>35_g?=TiJ|@x(KF0V3QktLx2D$J6;1qE>yMR5bCU%#3LBX2gCsBW5updI{J3 z2f)IK^o;1xjChKo^o;1{{2$JU7kKACn-LY$K5<5TorPc>+;2wA2GpAo$8-CT8Sy11 z+*oZ8F5$$R5ufJtUzid2SlW!(^syP?#hW{1K0L)h{5R&qs6*$&UDxVkZ3-;E{9dNT zC(nU<|K&OG>rb8oU*SEP12-;vrZ=E`*waXj&^Du=MjeawhDB*K_2NuJQ*UzsP5t^_ z(A16C$gx;Q!BCxFo1-zvUXp?Ty@P+)!GH5+;42;caKR68@G~;-S2+0V9Q;pi2mZSb zetQ$}FEv>G;SBsZ2Y-QsKUeVIbMTJ|{y!XiK?eR92mg5o|GArhALiia3;yR0zVoj1 z_inw~zIP7?zW0^efIr5;PZ0cB4t{9{{%;QcDF^?!;E#3iBLsge@E$)i0Dl*I_`8m! zgFxRrkz;yM)^Hwfv^eR%veVfLnf{ng6XzDP=84qFPjQ&wl_2D#L>VG=VtWUTkCl4h z)08lm&aC9;?5c3>H96b|NuSN%Z-gtNUL17-Sc0{3;_NtJv;6(-=O*({0>eEQBV-ag zV9p_Z1jnZ;;>4kO`$QtXnc6g3@{xk2tkutk^O7?^O%RevClL@2H>fGn!oW^rNf0pO8j$(qGwfdC{iSD?jKU{7YWZgNkI-yRv=@DM!gE ztJGZ?`u&OUt`HZbzDg!NXdXh7{+cte)HVW4`YQ}yf7o34ghtDD*DvmpeYziiYSAE_Jb{{^*`uNJ@bS2 zQ*A%^2&lYC^Yv)nl=*rzuWr5`%^N)*%VQz@cm3TF92&rYWC3peI}Pdj89&RO^_IM{ zW)=4>dF9(?6>q*hy$U*J*eck`fmLwt@0isa-K_o|3%IjlmaWPMGw=^M_*MtsvJm*c zIrz^BKF`6=&A>M}_!}Jj?*#v_gWt`9>D)APVE(2#+k2P}jaN>9p*aD#NSv_yABAfx zsw~MV`5s%}`<4uMIdZ;S+iS^21(>L)Z0s2nXgZBpUa&*55aqz&aL47D2{aWQX?KVp zMq%UE>8%=_lyTC)N#t>+tNt}8+003bR!AGJ*Bv@np*a4;xwl$Zahe!xFU%*wYm8Y- zP+RzeaDreP2sxwTds^Q+(YI%9FBenHhp!^};}aX->JSe2`LWarINSy5N1c`H3{C2p z2#ffBWCTTgYNX|nz(B^CAi07H6f_-0Q0FaIJGD7ILl!iO3SzYIlymh8BaeDlAmfkTI9o5uUAk+yMQh>(o)Y^oN`f!X68BX{we^@pkg~ ztYbC+DPhMt6wOkLuC2B@Q~=2ltyh9;KYz-31ZH~Y03c1 z-~|JXFK90GTFD!!le^dQN7Oy1n#P_vaoX>S%R44c`*U%5XIsTpmz_Sltzy{vuC|I3 z)^BR7-Fr-jukD9>XK!sAGpmeX2Nio~ceIV!d(37#?G`|{PxrQ4!P~kASnZX=>gwU< zbsU@JYx}Gn2iftvS%d9(QI_BCUzC+?*R!$?v*VAlhLFK8Cw21rS5seDPsqmuf#wS| zbn)(J*|tE_8K@GvUgL*plA3kdOZ;u*?=AkeYc9^?d;8Hx`?4MOl|5}0N37pDan8Y< z%lsGXt9ICV^kUz{Ewgflg~qDIdym;+AGAADXdSR44Km1qOq_OCae1b2a9c(8`n{^M zCk@BP?LDT)s_M;Dl|2CDFb6Vm+RY?QNIyq{1sdsEr&dKXtuhEMz2*%7Nw>;-!rC*i z5^K*6*Agnp)}Rry+Iqoz%G%QhGK2{hY^(Uf`ZaA8C$Hb28PTD+K`OnCf#!RdaN)9D zfu=uNFo8Rr$3gs!-OC}33p7TfP;)r?=m=)hJ9s`C;ma%bh~!9}h)AB#$tM%Z8r|`T zHz6UlW!YULRS;D3!s;!)IjdEF-x z$-f^>D-R)(e^J^0K_dC$j~tPFmm~ak^$90Tazyfke|JRkEAIG}KRdv0qy#7;c~k8_ zC6Z1Fs%YqjfBp;f@mID<|9_4?V&#)24l?@RAdX)|-}0~0!_TDY;gtVv^pFYJpBR>> ziJ=m=hDnMIdUM2ZcipK2Nmqs#evJMRFH`wk19L^s|F_V;?~DG4=s^la|3nJYs&Po* zK>h=I_Ak;yM+HR>9cBE#^za|k!gQc*&8}jbfP;>lmh)0!$3H zyCocMyx7sb-5ibXoqSP-oWY|@K9lBNBJl&(C<%OH!B`o4iEys9SAP0}M?Pu+R6Er% zFX<^y-XrcUof0Lkc+Y8BJW|uNXvW9pry@(W<)0(d% zLgLtN34HphoWR`u@TsxnG)a8~Iy9-zoL%}V)F_<81#+O3<^ZtsW7o~(Z)qK!#f2irMkhq1}28EiL%?Q8!nY{#~Y&iRvt?Kta)?Z_3g zLzwjA*lv^ZK3GTmpTKrIsvWk=xzNLQU#k+flW0V!Y>tOwJCAb@^_#E%623F9xHZ-- z(Xtl9tUFxz%K#r_e#yGr;@2(cfRpQbks-wi-y{{-Qs z(b@of2a3xe%>Q5DyVIl(slD2tn4Uvkkka1U_TY%wPDk#o@a-L2Mq;=aG2T{{;P(ig z3@j*9tUB}>3Xk>$&e>0dS?w=kH5z-^<^7f%YJ-96hvWd)Itf?Ac+Z|S6~5N^3yp6L zAsajDaJJ|K8n?k~5Sr+Z;Tf_U8hvLh<_lKZ)QvwSf%5#k< zrx8myk0!395qEoyP|RmqaF`Z1jV_Op#rR;PaVsfTpY*%dv{b}jsId0*SbItXjp75f z%k)6kut5JN{j)Af;F}Ow@ENZ~>Xu{vQHwqWmX#Ons0hsBCv^+W*l5ilV`|re0nPYF zJ(IHTEFmWb!xO^Ah{oI;?CYL5=Ncn#S3F~6-6JDNap3xm9^N0=N2;q(gGM*KlkTQK z!E`wo;_lJ`+ zanaL#3)^pDPyr1{;e%^dOndf=dvz~ojd+oK>UVClG z3-_eTvOyAFIIFMh2;S`E|8M-;u8w?8mMx3<2(h=NyNL5p6o0~t0KkApfXp8xtMge? z6>(#R+E!HTpK*i^)tiPrg|%C5GxCxIY8) zI0y7!mjOFN(n700isuS~$DM>W3^Z2AG9z&>$_|FWeMqy!Q{_z7^Vua!gUNr2k*)pGXZ$W~c<#<dw*ckL;e5UaaCw-_zHN<<_@+qPCjWVf$_M;Bzs4wfAV;G}1BZXpE;M)^;4tvt zD;bTTldOLfZNnQpej7ZCi|RoLG@hO+ruEOR?u%06hojX}^F2f5k(~^V+>p~CLXY&= z(+O_GJfbrxa~V~V4E&F1%j*dY5!?rSFUo`9DH|eneM4lM4H3erXo%c^$Nv2ck&N*{ zD)UKYT*kO~uS@>MAE(``ExLgl8sE}=s@>UbR9ke&xWHK)c1ydPo=>AMpwSm(Hab3` z|DAR}CDnfay#43-s=zT^2?6lB0#4U%9|g8mR#DjN!_v?9Y6lw?hSUh z-Xi)4CKZaa<0UmF;vejOBM|<`Iwn@bV3C;6HyCcMa)Y6*gPrWD(Qso`q-`V5oV)wqp#2{Vr2Ve~z@tSDb|oCI($~lb)X0e;qW!&ID<6~5${zI}H0s~2 zMy8V@HHWAF1=iN0|D#?J^yO&$ zT?cp&K+MO^3-9@^0naxMc%D1pdHR6o>hv?L(`9i!Ah+y|uS#?R(QE`_&&nB(welo^KrRJa@qJ^a0P+1D-GBxnJcARr#q@p0J%7V{|S? zE!+yJi(=v8x*GAlS*Ggu@&9!B2V3$VmQnrgu6@{%&~3fyLkfqlKI`%6@0VRPu>S${ z>;D1IHx787JK%ZxfamG~&lmE1`1<#+{+G2dV4bh(TkQ{9e@p&Ds{el1emdKY1hE_6 z^O<8IyWwrdLfZb_SN{uD9E{-*!O%3(-wQN4n1Pmt{N@ly{1C{`4uO3C5XeOt5Sy{w zkQf6P?qQ=LyX5si)3^HwqPX^pI&M*i>oN|E5?a#bAIyg_-@DfT9E#UN6ZChz)fuVj z154xUcdthhTo$f*K2ozSgy*$ySt$N`V%AU}XZMxGd&+jN?``cqN;^|Zm@W}?YX2CC zuVk*n&3c=6isR_A_=nw}mB7)bA0SEPEMToARNT7 z>vqQ1g@(OahV#MnmE8gFeSDn(mXyVpl?{9KQuah~{^~#ZFKbxVQ@5deUjN!{wc0V= zP`j<6`0g^_GXtxZDd>-EEYW6%w;_ArGf`Rq9fb|oOqGn_2)hoR!{hL*(6bJZ2 z>T_^leE|HA>J#tWy*?05y1t5k=t~Z$i!t`@bcGJGzS6%!`TwAQp8FX6!@-x+SE+lP zz8PO9z@x5vEZ)g~r+<2^f0k>{6yg9q)LZ6TewbPn_tmO6T`P|OyN?t9mgJt3(LbyI zb^rVyp*I}akDr=yWdHOz=Ez>m1xNPNrr>Bk<16gtUZr`fdC1lurw~7g7-BCM5OaiB zD?|?O5aRMQV$vbLLd;}E~?5NF}|_2>p69?&$@Jmlmw;)xFND-Q9eLhKUa3l4Gbk5aYn z30tj?bcpwS55)I`c#lI|lSX{rA$D+~d0rM`w-Dz##6@YuKRCok9O7klAWl~EOmv9X zq!F)ni1Qs{R4LxA=Fx5zD)X&0V!1>7zC*nBQV@SH#GwxHv^3(W4snb_oGHYIg!qQ0 zspcX6G~ytKc&tPGYZZup72;nV;;TPQHP0*Gu;zJ>3$@-T#D|4=y+eF3jd;I9Y;%a$ z2yv7UYaC)cjd-0yyxAc>im%?IX9)4%9b#1)@w*Q3GKYAN5JN)Db%Y*;-|hnB~|Oq^xlro|m13_Dz4)n5q!Ax; zh|fC2_i;>oG>g8r_^H22W8LJie($h03M(KiowhnHjrCK9b&bRN54=7e4GOCWtaH*> zVTU!*VNFoca=~IZ+EeGTZ{&~X{G<4(nsl}0ip?9NjzZUDTY?;VCUS2MJy&uQZkVkN z9XIJc$E@PRbDyB`ea%B{Pscl0#W;^1;cs?+xcP!$IG#u8F2Mf=8MiFnJB}rzQ1ed< zxqhhq9PCzL6?+QN2}{AjTBfs8iTiSvju@o69P=R5h3B$kx*4vMOHR z&+Ab0_a=v$FP~ytJy30N`bq8gzVHHOgO#NEG$Ov(Iif7%^ip1ya zeoPU!$8xKn1fSHE#qq&h<7?RqMh=nY`&W>=e0D@jG1=DTz*4g|R`Y!PrG|HW+Ivt@ zCb?kqLD>j*zX9%QaL-PRD$=UVglOVov~jmn``pBsBNgqzIem^PLRRVt`v~trv^Ktx zJ#iX1WavVUDYs&Fj2y$fKb-kY{y64RgSQB`t@?)ZZY()MV5m_8jbmk7v-NCm_rYM# zdB=+zfT*5w;c)?g6i`4(k;#|(+h-&Jv)_Wu*vGG&tThldMh6|r@!R&*`JE~ zmL>NF{dailJj+;Ny6_-2jLp7fGoK^A1USzeisQd9tNXraa~Wo)3yPb6?hiGen|*d7 zl-iOAOS?ush1i7Zx8~k=}BV!Lc#=_0tqM_w|)qFvz zZYFSJ5O}~v`Qzdr$UQGs^Lni288e5c0&g=YtD?=BRigUTf@6o%aIs7g1NsvIGh-Z?UBwm1<2ckrF0oR#F}N}e4>712L6+-8qmU@Svir?rQ6U8I3YBfHrYLS@+07R;t`0Vwp5@?=4 zIHN7HtJ}$z8r0$Df+C$~rG!M$c1Rd&wH7RF1ukAuvIz$orH`jvC81ue9TEgWDT^u2 z6c;qB<1`KAQi7`u*bk=k7pBs8ttb{#4~Xz4QuEmgMg$a`Z*V44rVM=9-<0V>u-u;6 zXTzM-W8*fF^9xY{t~yI)fyC*B=mNJD)q@&|ug4zW{pHN|OzxZ1YS!&X&6o`@WY-c- z1_ectk~ag5-ytVcolw-0ofof7c3k|bO!Ddekd=4xB(wict-${Koh7e3klBByR$y@a z&XN}$$PB+zE9^3Al3qNg7ey5^QMSt_7A@H^{a()$-)jA4r!XVxg-n}O3d&P}k|5X~lGyBFbLxw?0H0xcOKSvO@%Z*%y$g*?T zav4sHnyoj}!iNk{5jWg9aoU6@7@(uA!P4{~;?;Q3S_wpJG;W9`WCmh(AFc_Xh1XT>uXJ2r;Jz^kS8UdaCW;=@ zfJTJcEk^}DclvH}$E1~}V>U*5Tcf_0ZT-PtRQsPsSi}%nA!0pUHzL?9f^FpKw2AUx z>Ll1pINpal2xe_^N5v|SU|;nJcF*AH3&AhHWUu=`_Y?Q`ePZu)#$Dwvm2UaOg9-se zawL`9G-;i^Kj_{I#&h08P`n)VcuB_BgZh7M1#)y4Xwv#`SCAjb0r?Iyn9TwbC*2gl z@>2%q4~g?Ld0={FTeO0e(7^~QCKN`Si;B|rwHPNMiU1}=;5zYjhpZPy<2~fij?rx{ zG1EqFIfNOx4I*j!Gfq?L_|dowepKrC(YOqLROKkY}-GFUT&a*3ab4QcK&t=gA z($CiB*}hq;Dao)utd&lab3+HUxH^A@C$Jp?W$U0?L#U1jsJ04Ggh*{ZhQAasWk1XVDpeiC#7`7M@E~jxPlA1V!hPVEhuCu zrLrZBINu@O;Sj%b0f^TM(NijakVgEzL%hNv{`y-WUMEG6X?#do8tdy0Yl6etJ^`%X z3QJ1mXVX~6JFHV2)``mW4q-{D{AgmT4m-}YI`|#d6H-vOfrV1}!%~#W@-YhGQJVG5 zvTt3~a8(sr<96erYeLOqD?(?Dtqe5f`cu(6>axFG_TBF&oeBk3Ye8*_ zuolcQBlYTt#~Oy*GK?lf>7p0<+V5RVq)4Fb>+U}u{c#PzP`uDxO5LR@($H2ID_K$Z zT8u;0N|w6uNDIf+!pOr>Jl?i;|JHTfP?=jp>h-?-erEZ3{q?zYe<{$sYU7pxT+gjg z&q1UpC_7}KRquV8-lP4dSL8gBLZ|QQ@PzEip8iOhpY**KZuAkwNg-|bpztW^hn&m@ zMX*l4$;*7m&wMCw50F7NWKh@Abry}U5>ks+3^=J4Uvaqlvt#RU?VV%o9d+#;jaz$1 zgmQa&VXHX7P5sJ`~_+J8yx;Z zhreB%=uC${MEG0JH2m+S@h@@s(;R-WjCFYq|HZSwZw0?!JfSHKS8)V%(-N6>;%geN zn##kgJcODrfJa?W6=*tOPZKLcXHA?OX!<J;c_1g#aTzYk8`_D zX&t9%aYtY;9;f7XnBn-lqlp~XcuSES^xNma1OD?Z31wvAo;v$Iz2EO?Yrm(R{hoUI zJxSL~rJ3`e-b19JJv&l+{&9_qgiSp#&9%7$HdfTHV@6>g7 z$zMYqe?h9_55g+c@hQ%d*5Mj9rer;JUc=HVDKH#zvbX-((CkAB)*o#N{@e8JoLR00w(M5gxdR2cESz&GUkgp z$ZvBOw%Y~Efu1^a$(b8zki6U``E3P$bs6&89O*_S%Ot<0sC%^Nz40!LGqgif?>?0U zD^NtojZ%f|NPJnOVRMH^ypyQ5>XR6iP<4s1lC)}jj-Mhx{DP*=MgAQ=H=3wMn7=xI z$W13P32HZHhJ5M<5~FvtrPzOPMA>=h`zY%BNJl-{04eDSG*nYKSqKV!b;Cxs?qawT z!U-2uC6HbF=WxRwwox3~(JbrbnKs+iLxLuyDpum$94s>EAoaW^Svybm-!@{vDWppz%~lIir0^e@{qvNaE!MB9+tyW($w9}(U&1yzcwGEt0>|x7@la0R=;NX1B!(LgmE1YG z5PBJJ4c5nRG@jHpVMU7&5%wn;fhKS!SSCLm$-xX7S&pA_h?fXM-8zt;Dk&@vXjqu_ zIQjZee#&-TSuF7;m-S(^SU8&X8e@u1-jYu$(S^D!2O_G}-oD)5q9%#PpO3`deQ!9Zjn0Kwo zweKhQDj2Pkm$eT!QnI`DNmyL4{GpvD2A|>{dhFqJ_psj{#^@pMAxo#d`y2A#@Y2fo z9G81zCO84v z+@qcxab9BToGNHzPBig7$4tKQIf!F4^P%YBzr;UI+MVVfBmW`)P^i!p|M(&IDf`kP zd<5k@vwsf9Pj37F`H8s7zvd@j{@46OidAY-jC{o;~;GyP&37xatA#v%V-@5%pPmi&LJ=?f#%_+NJTr4Ii$!cRE--wVIR z;rr6~yoI9p=WU5+J*MS(jjqA zuX{otk+}Q3A8YCes*%N-pMZk`6>L+-qc=2An{kr+ZO*JJQ;Ko{KhDpdPOGdQPhrfqBR2 zB|S%jr+Y>{#dFmA&U%jL@2AoUmxRgWn)K_EFg(Boo9+I;+Laskp2F;#>P-3h7Y61ksA#H3kAqCR&I8a{blU!2 z_3>a6KRLr8asJ?jjan=@>Vi)Z<^bzW!hdA61_RF0AQbe{IDA@R?zZ{hrlN&3X1piG z?91Z&@Gj{v<*8QzEty)MHK}2h|g=<)b^$rA3)OQwWsU~cGcwrko23Pqg zI*cG6ZtRB>6}||obR}2u5@EyyFkC*7>@FwX7T79NI(xtewP zNDZO9R*Mp*SsQ98X7_3@l-U%4$TweCR7bI}Y1-k-2(JRh4)TUgL{MH_WZg9E#qi*% ze9yCW2y17B_9coJ_l_1^X)OfE`s$5u7r$`YiciykZ@+Lt2Ukv(B7zQFCc5Z6@ z{Ad+3;inXh?)V>XeLtKi!&3?Vs`n`hAULVDal#MNgDF!S(-^~vZy53ftCQap6d@G5 z#BpG1ESd?joCi8oqQu$M6pBy+^uY?lJKO3w1sHAh%PQf4GmopD3%utnr9;E4_TV`l z{AQ(<@DtFuv8YP-6k2TzY^Gopg2flRgEA81EdL4a8{+c9rwsRN!-;{evy8TqZ0f8} zo$$4Y*9tW&cGi7XpMILEk}RH_!L(3l-pQ7VI%=PJ@q;)fN18d)1}*O$V8ZNrWM{ao zv#|D4qPSZkyASl&XP{TDV0@%~<}>B&VI5YpLM*ic%i44LY#5Ji(IYq78E8i-mHiq? z;qFhi=woaHJfuBQNaTJs8lD#=iybdWaRCx7oc;4K(QVjfoKIZ}r+-)98HuO9^C97} zd9sl-v|*i~+P^UUd-5+#|DOB{)4z*;;q39N@Mlo5Y3gXnO>U&aVhU^XOdm8edOiB^ z(n!fmb=#b;fToDjT}cd#w`*ZW`zNNtI_F$7l;nnONqL3Ni7Ny&CMH`!@>a`^JC#q%eq5weL?PI%1tc!ykWXwb8+6V z_OjxO*S41xUi@l%Sw1I6mjy50(q5K*@vio=tc!cA@0^u&aclLRwOJRh^tP+^(R20E ztBa2z3k*!Ym=wWdT|`4vP28o5=}j6EoayJ>P}X0 z>xCi^sDBzQn^iZv>)W7Y*cU_vu19Hax?H_1m_7^Ppm2_WD*)=g{lu$+1;1{uIq}4T zUk5b`L`!;6eR)y-MBb1!H5gX0PC2(KsZ`E^#!bkuV%^iC!?vHFD6g}Ao^Vd0AXmPQ zBBb*ZQ)@AW?#k5Hh&MU4?jTVyXH7rI(Sgw!2dxlKTOn=;x{U9$j6sdoWq~?oI+(*} z2(bvn<4-KQ55$7}iA6V9;e`E~-Lb4RC|a_;?v}0)rBi$N4bDv>eyI)7=6N>CsILYm z=32=0CQp8bYlR8Uq+^i^?E^P5qn_KW5H%K^b>5T65g+=ESmX)zbssSz>RT5{M31*Y zSQVAI>w24J2sse;y@IXYMyF7iN-=wB^?#(Pw|(^9UUupvg0UHM)wuI^(&_gMr2If;eLT`%XY{^~ z`psf|@M8r3X$Dez*(s{>^(J9)x?1fdY3@1+aTu&=K>I)>1?MCo<2#^{f^hPri3NAG zmz^ZCJ+)$;t@Ev{+aMo8jer#S;+yt6#!dI9Q|^KpFLb`U?Eld9vjl88=OxHEmK8;? z(^;;f-`HMdc%N?1*{d9(S8}vh`*~I5UhPkQW?~VqC`WtQXC}R7?+@C01KRfoLCX9- zEA{>Ga|V3wD%8-%ITX(ZnrbDPzX_2KXb*Ybxf!17&Sp||5(<&{My}O(NY+j)Z&{d} zm8)Ofx7vNH-FMi12lv<`XJJT`*%(Xl^pz!3Wwpim3)&Yi2x|<8oVf-PT13#$!hrLH zNy3HGe~^*0xJ1yimwNfvPOK#7+KH1}l-qFS=87dSGdAE^xx+30FhTSfDa53I!sOSi z+7mPQU3=M86Pbv(Sq-Vr<^yLI233+Q>Cq84Np9-UWH}Il3XEhG@dFh z7La|lsuX5SMG|>AbT%=}U#%W_p!l-4)9VKc!hG6gpCF(c@wY!xBibv}62cLH2fedq6sUszea_V>kSF0EdBd-0j8aE1QLN`LaeIH12Y zXrlPcwSuk{bZzz8CW9U*JadDfuL}BV_1cdLmBN<7GdBtPmY{D{KQL<;QqBRcr*J*U z^;E9CT!-g6LC4Ci@~C*PtMG5fO8(BN8K%$OWU_L!1c`*UsJpOlye4NBza;qvrA&>; zbspDf;*8(@$I)yxcub8+OD?3*hR*m3Aq~Orx(2A%-||~5dGG2aXwtKHupBAbyS}%( zGbLY$OtM^y``y<@(CPJp3=OqyF$w}XO9)ubM7lMwndMPaNQPBvkUAdl~_${i!JWE8Fq zByJ?}>W#CrgQ1oKPIh@+2Ii`oo*cvLTwZ%<>Kia-oKn4Jc2*_*U6!-fM^B!x zHY>R&xk`SAIp@Ix#@x`}HfYkz$K_lM~O&YHnRt2(`#t{77w8zCN{B9 zRrKWHf0nKL_J-;QW@k-Wojkc>g8+w?J<#wYrFs@Wm$rQT^p34~&r5G$As4}M|5-X2 z&GE^K?Oj)b-Zvf(D@C4+>oj~9Y3kI97xnW}d)evV)jTt4ZTr}(^EViqoE^zvQeux< zPruvv<9=gDy4PBoeswPkxPI>zf80NiKZ@VfLlFNpe(3Tmf)lnTizZGb^`eQRl)B@G z0&^Y^;90q&Jbt+DpYg-CQ;Z+B_2Y-C=gm+-pK7g;HK$P6aKnM@8J`&0+VyuM5-?KZ zF~_g_76yd)H}Pqf>ii6}`SLk$IfCW)xbm^vu@(XA%!KW9u=g`zyBut5CalN7EF9U| zESKJ12eW8oDcF7ovtVQ?*g?Va=JX|(HDjIlYfvl8 zqJ!@?z7R?H8wakRl4eUrnit6PI>Hq6jzn;`{VS%^34ga$VnU4 zVWk@8i)ERzC^qU_g)!n7mBSei3ef(h9y3CKY~Hgza&4XnL3B^d8f#tFgf$syiY_36k}CaeF(-&vkpRkf7A@{6SI6)$%jC()Nm%EcH;N?G-QCT`xHtRqxmg{$y2?&|7o$@!NgLd8mV^dnxg^?Z>El zKc4lX1gt<}A*{{xJy{p(PBy_w)9Vx(7T+(;t6e`5`%BM{?mTcA(C~WZ>{o9$9RhO@ zb#IKRd+Kac_wHn^`04h|KRhu{>K=9g)AhDZ&7YX}>tqi;^TR3-yl-;H6{oXp>@i)+ z*;B@$xqa&(v((fJ4>DCv&=|bWl08?aVE1to`$m)MstU97&ULvx;wIXn7l_boMOXT74tQ zOfELkypeRb9XZbVc4`U@8d-OUW%wSk&l+$NFVJ9f#a}k?}v=yna9kDfUC|+3>7L7$rIW6G9>1$_4AO0Edr-#l> z|6u5N2RfwN%P75UHR9f}wPVreezN2P2kLMj^tqq75MSkGTODXCGvcvh%Jkw5#q-J= z+g?3n3G^=aqxIA89YEX1NI>h$&rCUx_F7+Z5!Byp-iCQ?hEmKC)Fr zdA2z}JT3AV+kPjze4e%q8>d2ilHajm*R3~)loaU?o>AQO7sx)XV`w~}!`-2w&|h8@ zeTEc*4@%el_`Q8O*gMbOVDI*~pNRJMbn+fi;z^U%0UQ%@v?`e4fPWGtv}#xa9x)4d z{DAM1sshn4(Ln-D(tkvGWt<(|s0EZM;hLUE%?`JA60Knqfo2^n7;eEt z1|8Dz7EB`XU6El2!|?-lY@JQ-Qjpuwh#j zQginj89efJjr`4LDW4jHQw<+T*t-Vz#ECQ)`t^QSS!I+O!`q3(OVZt zFKKuyA^E(wX#3v!?6B2B4Fa`Qi?{X-(#^!vZrJZ`&iEoXoPJfMMG|R7aJ#QPC9Ja1 zZXII?-G}ArV%VlvP0T!`nXQs=U7;K$Mo}l4mE4`pheET4R{ty-ckwNb1zs7#;Rv*=Ye{y?`unaXZuD->BS27_N44$M5#UfW^*Ah)q~_ z_wKJ}j&H4>Owi$aXFXxoK`kCskc82*>`on)p8GV=_|_&yX}l-3XpqajTQoS@(+A@{ zZn>a_f-zySLhzz+)DyTCZ<1@V?J=+YRC<{p8@(&dZ_?`p+2YQUX`{^-Vw!OEjV1gy zhDvU;p)|YJTPF}D7Eb?`a6|x6-`bIIlEp>!z$Iu;er%!rO46VFm}LB-$LRFJaLIu> zh|n|mu@s?%YCqSX-<07UCXLqg+;w$Z-2y=s`NZU&oM1vw&me%A#HQxcOTrs5C80O# zp6P`Zfo|rY+uS-L1hbyJ!p&|KA#mT_MfclBoqR;30p0sLqBWD5mh>|00TNROS4DjL z&PmMrOqED>s}f<~G3I;G6FW$BKp+Pf@opLbR`EWA2Q&)zuH@>uW~iQQ80xK8X7`7Y=h?xx6e{x%WMMlXS- zM18BB_LDK6V~wjc`=EZEm2Ot9v(;6a5~|vD98!ntaJ(aD5&(Z3DAk8BRNte&dUzS1H2c`JdN*`7s7V>*IefVYc zVTMerazkpBW|UR?8At+s9(tZwP;XRS?^y@mH|wB;xBEd1G|E*_I+<}41~cPiwbIc% zgmqB+tIRscIJ(JZ0plRp1~;2+@csVfKkkK(k|(@?x#yR~FFaGd5#!(A%@}yx*jW1w+183WHyd1MECM~4W05J6RI`ntvjmCT~b(UQBZ z1?tSjs`{$bd4=fc*tGTlON$AS3|J55d#-l@*C9(f~3VWAKVtx@mQr-qu==i|w zW>HLhZsAh4xjlioHTurP%EZ?j{A7>819Fvb6qw`JJ>bT6quzq*%fpx@j2+y7Gh4|_0etTa;$ z@A6azX@=LTUdZHkF24deU2C#nVlAl`OdQi^{}Y&lzyWg(xbfR?iTE8dQ<~L%J|n&M zHnf#`oWFDP()>=`SLT-yyEMF=K`QDQq>S%LsOYq!qtHUo;W$22NGFScDOMsH^dt#ik|~Cy zOiLrTr1^~^`j+3ju)Pd1F$(@f6ChW7P#5o-Y-JkKRDe#?jNWlZ10i6;iQ{hKf-i9t7vx93 zat7wARq~K&9#WY`oa7LH>=0kh2l3KTAl~T^PfsI`bckae;v+)*fe>dp#80FVvmN5` z4)NDQoFYUoCf~--rpox{9xLNkE|jrOh(8h{{>&=lk~HGO4sp3doFc>@3(>U6 z@Cn6f#4kF;Gacer10W_rwD1Xo(pXszE7xItE(q2$!czEz*YZ;hvjMq}f!fW5`VJP> za$zZa!oz8-ha6VYVRb6U6<`rQ;gsHey`9JH+sE{aeBICAzY|X0Jg7nO_&&(hkh;yB zXrj$O4Mt0TjHmXI#|Zk<{BX&Sanvs2iIdQXz6W1Dw=mE&Mz5Xo{v^A-N`Ct@?bbZ; zf5ok$_yK*$ZhtQK@U`4VOMXQ9+jNJ|V)Hq;m*X=MH79icu}b#Je2R4vx8_X}Gtcoc zJjP0caesl3;Skn!wr_l!`w!j&mA3U*ux%xHTHB-DF^%ou6s`pSiTlf zd->WIOQV(^Ap?*s6oeHcZX8a~hJe+AE8!Ia*S*A-ZohV*aZylT;|dsJD^+v2I9l@! zf2jH5Y$n%_h)4H#0-xn7@Y&w@QoQk3`&NfaRuKEFjn%HiRDIigO};EA`C2@}S!VJv zvLbfKsXI6>-YtLYm@^Mn#A@)$WJ88`zLnd~hnW$}%`9~OnpLXJ3adbyf*w(%QrqI} zT+2B8ILhfM*br7zkx;yiFcY%;t>DSF;}EuNp^}<5F207NkLHIctkog+7wXotaE zqTvkOQoBd+l@TKw#b}9&*Q#3BvZV`OscPz|JGQwybEYV>0&J`EXfF-A#Q{bDh$Kp@ zguR-iMDOz~$+1|NoYB*EOM_&99w8-;tGrzP%m_v)`>nRJ%pF!tN z_>~YmN1bY?XL1cRPCkk!OrA$>HJ9aM&HVXt_?GbQgWUmV50F9h<&kT5?-HeEV{R9b zEuk!ov%jnPEpb^SdD9j?vdesyhbaxL=CWGctm@p)7sNHVW_VXOvuK9=P0}A7iWivQ zZj)`QCp`zm#m>>0ea#ZC)t_3!Qfb(13hW8D5+|0BT0pL;r3i8zo+fOqYKcZ7vb}#` zaFGv>=Ok*g*$Vq|I8k0Q!j=!za@K7=ViDqixJrUNHJT7kL`%#8G0ck9=>z0%#j}%v z6620pNbvVe?#+cNdXv$TWT5eVveQ}vOs9(c*H}=7U#mT}Rht3~3^3(f_|pB^ei!lX z&R?y|)bqUfQU>a9pbi)B&Oloo$S$MKck!ioQFNY#mvZjhuk??1$4adIrSns94h@1g zPAfB?M=)2sHyw7nf@;yfwNYpP_Pxw%p3QgTS=hXc0A<0PU+8uU5fLWl{2~=0PtiC_ zI^}PONmnBLdeUK`>S3hi?mRQ}WL<^|WFZnS_3{ z*D{=|`^&f|$GOVU-tE`BJok=!R@3IDV&>WVgYLax_C3#p%}agn|55sLKh?_oT3nB1*Lu-#_2lQ2a z`kZ#;T2KdR-yE<=2(+TX|hoa({*YS^^eEuPr1}SoyW3#0;yv7I3bx za}7J!uyeuIQl-4U9+X=ytzJm-u!Ft)>Wsyu8p#UYN2s1nLu<9;w~rq7D*wI84YIyY zCc@rTbzKA@z}EXH2@IegsY;=16%QjAy|&Xny5W>u@k4>6XjL;xD zgxZr0aOYtTx(?f(jn5?gY5qHWyP0pze@C7L?|78kYVC0=>=-J|o)H+ivTG&}Df`dJ zR=~l)+;SR6Ol{`C{@--&6Z)~rOW%&%j-4F6!^I`e;YzG^h@!o&v=R-1HAK_l}U3n~Ps9vn>m+{JUmgK+E zGG3XHvAw38mSbCTTRZobW1AJY)#pH#qxaf!BxLBe_7FKH_g)hkALKW0Jbe<>{zyBI zOQWuymSxm^-C7PAiNWYWA-X)EX0NmM%T~kXSi^Bb*q%?Vvg}=X|1QAuzV@~~vs>Gj z?fF!yXaXelF6qPlwCIP~p0DJdpSFb)N9FOH+|%yAz{kT-@8NR1lH9s7!0l5U#+8KM zJ?d>9IH0fHU*_lO<6r=qd1s|2)|GxU5U#{_}{s5h+>8dFkm{evvh%w;_s^cTM;U z4dzVOvIEL`ce7wfY!l46g=KYHjYkS5T4rGWnI?RsqT$9(ESAI()>jfUN_8V)p%!5Q zIbno7W9M5HPE0NFvmvM~<-eVh2i*Lg7;}U9DH#7LvA9N0xFuoVK{FH{h%$ZHMrX~u zZPy(Cx&yg*ld}d1^r8c~c$2e^(bK5edSUTz zC(k;`i#NF(y|VXT#hV$v=K@VLXo0TJ zQ6eMI^!&)#NLnjzp<38iiK2ilc|_s#uoN7Y-7O3f5nk~w6v;s468cSOlE4@aG@Jt* zWeym7O7%|W<27M?&l_~o5FSDC_fj*oB1w<1IeJ_=sl+I}DkVxL#$2a`SPrvg4@kOv z^7QfFz;A_rKEJvC8@$+V??}kCS$yRVnZ1?rFk)Q0)h(81=PI6rd@in)L!{IZb}Gpp z8@m53exvtmb`L$seXbzP?hgXto?mcmZhug*TkIZ)&Fl~I3xr=FHm^U(FA#o#*sT5_ zzd-l}VsrX~`~s0jkpF(N^*_k7|9*nOz+JWlnzX;H%SSz;#O9^6(y&;~SAY{7A9zUH zw-KL0^A<{=n%63UYF-Dw1@k0N&6hki-{h(JCQr?mJZ0FqhMjBJxdrox!+|`tO7hfd z5p@SS;C^@^)Y0|x?X>2c_D=e)|L%815-*kwLgF&niqGxeEA&vN1hdA6J{y>Nmh3HvacMeHqy?wJ~J=h_hBg9EU z^enE=rV&><#0^|1<7q;y6ryKwy(^8l&>=qH5DSI)9U*!a*DKSAKXZul9Acgjzbiz~ z;yNLXc%DPN#32q6;w3`#EUy2SM*M<9{HjCTH3-BiA$k_qgZ@;l_p{9n>)ufg@hu@% z3(;+G81iBoah*eaiwm`WMu?Mz=viDBrxE|;5FdAli-h=nA$k_qhBV?Fhj^ny{DlyI zAVkmNIw_4<;Shi55T^=piV!`E>&P_XmmOlMLp)E2KNO;7QmmEPX~ZKO;-?+r2qFGR zh@Qpu&B3YWd3~cb&j(zn^%iXBkN!l6p2hXyG~z=JG3gN3vb6B%PeC+`YfBnyzQel1 zVLc+O%Y-G1>yOe{QykWn4(m2yO%s+ZuAwy6Sci3y!}{dW)L{)+SX`HMVR8N4+p@S0 zu(vYYLf9{^S9@?>#I@o7Vx-<%C?Vv$AkpEl2VR(8mN-Yz8bH}8)S6c!CD^fUGd>at zoc~G`(IPHqLO3K+L{(tkO{CyO<9N_+uac#8wQf<5m5~>l;~hZJl5;Se4mq*!Wz-^Z z6aaCC*0by#IYdfIv9^AbM~Os*ekKPO^W(>WBF*Dj_)?IS1|eW)QBOyj&npd=tO#6p zlK~NGWMJ*!jch~}J}}NYFC4h;H@x-^b-DTTysWv#-_%-n4l+le={f)s+=Lw_HYK4b zi;;ZFXrV_}kP`P76!Nn=+_q^DQMO@lv~3@Q>&km;ACt(p3Zu)W2bo$)MI=TIxSrv(ISlUg-fndl-eWYlDIt*vBvj2a#VKlu(jpJwmWS9! z^ZD7F^PvWi2@1pT(lR>{Bi0RuaMQ{Uxt zzk5$SSr%{S5RrF7zPCdqN$ks=?G_{?mrJF7f1}^mDZxQRJ-d0^5!g8-#)pOwsjPW{z}6&*kheXW;ii{ z#(*quDq@L3rmhX4_+*87nk-k?%?+D7Lh>c$ceUqcYJzdw-9@d`eGo}SYF?0ms{)Gl zhikCIvLX+G0hh#+0FnV(BoE4=Kdvb zSgJ1itz?@i?15+!ATsQ4n6)MInN^S@S12s;P`T1rK{aw>)O=uD^{mZqO!G&5b`74a z(}~=RI|A;s_zXmAIJ=pcK5AXg!x;9G8Ii+ndpiSXJh$+m<}=!cjZc zQon=OTwherw*ZC=T4#X9qAE*FZotv_uJACduCxj;d-_06Qb$7ZgAMO|UY_%=YaS|E zp)w959J1>NES$fs681E!gY!oUV@gKOdrxH313c2uUL{9YC!MPHiM)F$Gp>)LZq>&# zLBb94R#IX}@t^^kO=-fr;gHumi>YTU;oF4!HhODRFBjRz*WbDjsHh7k-RP~AdXukC zq(P=Q5jk+*DnYjb#rG=zgsYw(loe^y8OPYxNd(uuFI)<&FA@R=Bp9_}5;R|I6>lDx zqL8o}BJnMjYgkiKkr4vbX%c8L$me}ZFQyps;-X^e5{Y-RA^kMUh(RN@xML<;lK+IT zv#h)i71fgru2^RZwgi~k--cBp$H-5$CPD+*Y)Z8};o%%F$hz z+HVTqtx#SV^3Fo6=kq{goCrp;0n$zyqT%>U;Um|`upg%Vgp9U;p7? zj`+6S?nKUL&3ews(u!JsIQ}A0L5c4li@&Z`g}1(8ET8-oPngmEN`w=W61a+y$IX_j z4t4Uv@tF1`Wq!2EN+ageZ`o?~X`}$&Vw+J&#!V-X=n(KIKd*sa?>6iTuUILNo|Xl! zH=fSTL$)mpHexi+H&Et~DrJP*rE`j-l~3^&({2Xg+(dL32yjA6&pGSUgP({J3ls2a zz%d^ShxoY~My<*m9bw93egao)LQh4fn-ggl4>x7eefcSX|%{* z!}|F~4Ja)q+uc8-7Hs97df!&mN>fHTh9nCB3!HC65o5M;A5N4}lt$<1OnEJzgdggL zqmwv>XJ(iQYN^8RE7JOsp7C#oHoWytI8eq-R;(-xGF6@Tl&1Wf^${J z$oxzbPp~7CQd6Jh7-+1)cph!O49TO>21g_@wTT9K$ar+V^}5n9getZjjcnUO;K|Y%SI^;^jCKQq2;gMs-5y%?bNJq z=mS3NAdA#O6J0e$Rc&_WA`ut{bghj{0}MT8RjKqMov1=iKjC>=E)V;bE8IehUeFl% zI*7_v{6;WdQQo_UF~TzS_gprGOKF$9z=FCqQw9T#f8qOB4Nte(A_`r-q3PAx+6M8N z(liNN_AE2a|zA5`H@qU7n>lN=OIJI8!euB02iuV&7Q{R5qca~HV>i4d1E-5F} z?_IGaQ9}LRHExLu^&4TYQ(NZIB;7-ix0M&R+yRFsg-Uh=5@jgm5@Izu%H6c=yoCSr zhwR5{!*o>NzUlHuu`~3xMrd>V$vq(h)w-0AS!wtP)w=uB>VtaqLEnqlc%c@lhzxs1 zQb&F);jfP+3K}#9)S@oKzV;-)XoIa!ls9y?FTL`XXgzkaA3hqbcMA&*@3$|#{FYF? zEh$7BcqbIIK!qA`AiMk)f4xjb(S{!PN=q8S@&+8oF2AL`UW*w)e*-C!Mvhx>cm{v4 zQIu2mUBo+omh&$!Hqsqn?@TV9=qwKU)jd`UY!Iz>kFA3H4(|O}EOOaH+t7vyZ;$-T z%aP$AGD$XShe8pbBzFnVny*DDk|Jy}gr12>q?QCVTQX;9;q=MQx5C${5y^q}^}NsW z%CmT)EFg;~$^x=@qU$2uBy4ByUb??_dVHz3} z(U%}<{mMB z&E<$zTVB|l{OE;OF@{b&?WYd#5r8cMd@U)!PhMCKptpMaN5yBhR&U=@eC7(djeAij zw!dxA*Ne|wEvV;szWqgmK39C^dOX#qJ$|w1@r(VAUu<^#VyEL5+uSAoIga3$WyF`5M|m_wD!#-@*$*#)Y@Hon z{^ioQQFnf69nWlG*ifL8i1;v*ktNq({MN{|oVPH3!qM0#PWbu+4g~u~|MIS?WL+*d zXA3JPSDvs1{*?Sk3n8N}*T5Y9m*m<>A0=O{cn1o!@F{qy?+qlrNcPFS!P*5oEh^v4 zFTFGUisXumcY#&gBFkj$f}SMM8O6zulYV!_OME%CMHbB31^d%!R;SaHC3DmQB^hYA zT*k`~KjIIh&uU?A&MqteE|q^*U-?Jz!C$G?=9 z@h`VH*<$G`C&>dlCr-B1hbwhg41w9!W?2un{AAcg@x`#4=3bfYv3FvQmcnz~Y}Hza zLWZQr5S(2Sj(>=01o3eRRLp5eoxN-KvA@^iwh_TiO3a;0X{K30C!BNR#C2xJ!2Cgc zt*%Tu3i*u_?dTU7oaMXy^_`5=X@TQvDWidVO2_=75nUUAVYE_%gf_=B!|JL~?SQg~BbMgbd} z8A1`?_O!e;2l?jK_Peh9WAyecEx>RgrZyV~>)z_RBsJXn#+$V-dxM-p7nr};BXS7~ zqHE~8EA2AW#UkNG4#*?`p=V19)05_(!)KcLOYPa)eL%}B1m1sq?+;F1z<3N!UBEzq zTQLTMV-{$!kPT(+n~()t7|_UqEgkKfkOf;@oQJ*BlbN!hsn(p$NSOYMTfNdq*+wGy zI7K>~1c^X%_4mqlEyd{&6BqTU{f@Z6Od9}w{DhULmQu-&h!gm)v~oJYv-qbNU%*FA z@&jKCRd40b7}fQ-Yk;(TVDPTr1NB%*CVpV9S&nH(O_AOR%vY28-b=|xd!xxEI=9EY(R@)pcfa10YeEhB z=aSehZd#Q(vcRT#onzL}%lRF+4qapHx{rQrD5xq!f_si%nj4 z{lyu|frfjt)yyzA2aA*}3pCsUOpBar(~F#(2rD${U@nkynABn!3 z;b$LD^ldQ-^1~;XWisX6*$&Vr@4hI&{}b|Vttao+dh%|qC-08xlXqWHSv+|+LeThr z@@}-m`Gf|V-hgN>ZE^DtW{MPr51@ZfUvcWme^Gx5BPUmwySe_unS!pOm5ub2sd~ zJ0lJ4g^`lirvHJN9-s8dMHPX>#Wu-1YpXc;dP;-__**W<-zdvq$|B=yD}|;-)BH#o zlv6$~m6V9f zAs4EO0<9od+KR%a48WW%WfT9t0rc2Dhxs2--d*+;OcFj5ek!#1fPWzi&K9x z+Jim%ME8sOaQE0`^K!K0@+F<+Yte(LS_rv;-6Xf%3} z3r(Vn0ck=3t+{kDAPp*@HJ&a2IU5m}PDrG`7bpm7gpiqkz(%A3(cL<~R-{@_=`*_f z4j!$$^=^?$Y(-N&t=Fx6ltH;GQKc)xWS8r3a;3k2MbYW+Ur|{4`&Sf?{{EGUqE}q> zii=)x8U8C5r1b^$H`Bgay05<}ZK{=Be@Btf?*}K7{^;*{*5ALd{{96Lcnp=o>u*GT zCETg1vJlnE)kKiM{2SkMZa6w&7mJ2+#!Tqs|A#yiZuy8RxlT{RfJ`i0&RAl;eI`t# zC0}Q>gahHuQyO*xGHQWR@I4=MhXWGiDBnNYanEW3E~MqtHv0F;g|zdcsKxoy#oK3asw*Z$lb+dMCqvOk)NW_GARVj|tENiHq2&r5Zz zK~0YCSDtdpLlG(|0+-28T1Hc!BqKVM;&G{@L|h0*qRVjQ&KrJt!7p%rhg*3fCHw0B z7-h$<$uF+48+G2Xqn|eV>v6=dKjti9j?Pg~iwK7TzuTqpoXP$rd1G$cPb^C^Lh2-= zpzV1M676CC#1LZi&WQ|!mY)Od~FLi0iq~JTJclVh)Jr?DD%b z)}0RPL5KD9%U~TPEIGSeoyPjP!@AyK-7T!6g(YW~i_=)&bXb=@zsK zJiK(DhVa1sg~qN}ZDXOVwy4xRT^f%_ZK1P6S%lH&wc#iW3rcfog;sxNHnq-XE4Bg3 z1;__rw%lxXTonp3A`IwZ!$}3ptPtpT28uRMz@v8U>|g4slv8Z*AK?rl`?k?rE0)2g zW@e={Muj)p&cfbbX>b2ewc@bYJXs??PQ7rgp<4DRIl{U#eP@v~ArMr{Y`fj9 zS_Zl0pg`3!-=MoxOClI7akcy?Joq82rCPmM$btJ_53|g(-U`ZITEP%~Rb8wlH}jr_ zQG3ZMv0hROD{Q(ABv!~}KpCnHtPJseIzdDwQ!}WrY6dHR+!lDZk|z5iZcx%1E#bjS zRtU69M!!!yE#Dp*$)-QeHyE(Ajf^&!ptQ7@>d_F5R{<#Edr|4I^t{+j{34FKO|&D_ zp_*x$VWopZc;OW54f>LnQ%3|EB~NPUK5*YQwSrYRG3E%}mnvUYQ&{mI&8=#zZ$+## z?b*{CsO}VlyK-nTTjf*rRrEmOD9z=f5kcvWVqLB1f&1EIuPiilx6G{gbyczW!C3r5 zHf*Wa?&odykNe4|gT2K1wIfF(t0|v;^6*WmZDg#X`d*LuUW@sjQJf5xrK_TlONFRV z*#yiI7mY86JseOQECMJa8f~zU+pC*Y?$Y#fE=ykC@$nIzS6zd4!2fH^8H$lZ zl%3v{#ItBPhKT3_JatQ9S0 zaabTbg`bZAn}MRmZ}(=JXvwyF86`q3Io$iDgA)&UE$2b}5#qtwr6imfRopKg7xKr)@|D!Y`?E)E z;+@BZ`Lq-dFRa zk5`4CNp(EFDDYiw3U$Li&1y#GbS9;-2FMTB{HPrFwXk=N&nI-R-K^b2PTcCYt9H8h zd*LJBv!lH~;q7)^-F<}K$1$rjJ1tGU{ zf35xt+}EQN5G_)>ewBE*_H$ls%Q;Lr-o~-R_LtVjQ~P}j%Wwe<+$ZN$GJ#`?-H|}H z%2sMoIdKw{q2lvOApO1U6~3;R@55NhcJv$CCrSrJtt;!cOx})qtPYv)lY=Fn&TO;w8BD-IT z`QFjyx-W&B3-k#?v{F;)?u_zk{QZZG?_X;RcJK3bTOy3_Sp1p5+!5;Cz#z&!-*LB0y0<=KN=6?n+!ft`MK zU*N|13&guyOkZZUulYX`g)r5A(r<2EDE;PE26nLgRt9#k;8ua=CS2CKOY&m&i^S%i96VQ_6Eyg;Tr7Yz7@S4WesQH+8yL&DrUr& z`7qLab|HOzcD{a!*@Q9wc!SOeYSHZ|yFZti_aZ!J@6jwCX?|RnviMd=Holb*1svxq zUJ@Y~LgJFDSfZ{9;SiNEf5`kl;FHMBQB8Jpb^dX)xQWE~Ys-Fi_h&9`i4Y@E5x)uI z`eMZQw^+&ZfkuVRC~N2`oc=||D#Ze$hf{WN;dwdA?&j>su%(eXd+RAgpy73XM{bF4 z9o89%zZcoPHpUD@AcoqMKOY?&ELfcED1Sv=U3o9!MKavGqV8R0v7yuF*#1>}zutSV z4=NY?`ND&k|4QZJ>^_M3uv9M2>4RuHE?))vAVCMo>w{=3E;#vpkbDOz=z~CrNhX}m zK+|N#Lf4y6%K-nJ$BvGKSD2I^55e9W6v6sj!5h9uVYm$JK z_os;@hkCz!H`Z+Ym%pv6w~G%8N&Xj}0-&4I%D6ES8D7+#e>nP>NeM(BGmaU#rR$Rj z6dC;GFcgyqWiB1LB}FoGUx8#Y$8+sH4BX(Xd+>(~hVH>1E|_yq^|r~yOW*L>wYYlQdD%->QO?=%;Md0ej=+-Jeww}X?*bed z{95Z>^aP`2R(iCoWV8^5;IdS9ee=(u>Gh=38h>PPkC%?nY8U%6^F17%jkVgvG;Uw{ z4u@wWuy%3Izrw@8EXjf=<4FQW*L@g~9m!WhLMf6T7C0QqA8SxY@K0lNn&O7@5JU_hnU+}|=58;O?NFP5;LHhV%3ev|9Q; zn1b~2!xW^CAEqE5haX;S{P1GqhZh?^y!d~}52w)GDdNaE>Ttxd3%B%0vm=i8?Eb$) z9OkD0E1YnLV};^=C!Ebgnlz3-?EE;B96#>-cogjMaPwoaw?5CgIoV5d7CTn(g=c>2 zM*igV$bVDwWMFO`!^iAtefU?zgBvxv!8KrXv5!>xpneZJs2?=jgAVQo&GDfAe$b!? z9nue)=RuF?2hI1OL;FGd=turzUH8Gi2F$0`+ol#TjW!(MGj7To4!WDlhF*75)W8+q zy_w%X6?gM{Pw@_Z7Z>kyldbXVscz1#b#v~R^qhNV-<+#chq}JRk7GcmL02gnTl9&; z(b$a!JvXRO+1GMk7Jr^|;*qn=R!}JqRP3AK<{ytnL?Y;;C!Bp-kFUq#Z$t*$5+z2u zO#Z@}OVIzj&hI~n=hdCa!8!}DUF&!6U^IxHHM}T>;D}8oyYBa0LFH;f<02?7RocBr z94j?Sq-@9bB#$tJT+-AzR{obp|%8_r= zUl>0xN4`x%jGvby-=-nP&&!c-(-7n5<%{$BAjZ$jk#EyD#?Q+a7xY2;#^XPfZ_%lc zZ}G=LzTF?3jC{L4IJ_bG{w4V1T!KH&CHUi9!YT4BP&#?H(v7{zZtM+DkG;S4jlCx| z_NJ%D-Y|_U7nX0zP-En!h`b!&POoF8APSYa6-VveMbeG8M@nXd15N)9B-}h>M5uW% zPPPvqH=TRdD1vvK3q0CQbt+Si8fu>dV{-WaOSr-)tnx5b(HS}~oaJ1%pfAcljDCBJ3T+8Oo zfA76zcW=06H94*d`BsHWI6Nn|woHrW=Yu~^D&2T}==8Q&vyG?_yoAVhwiuF_fXy~qM%VxqDGqlYEWp2fF_dQ8J!4LRHzgwXp6skL6`wlE`dpe zaU7*;TWzh{+V-c_R%_J)N=YC?0^aa`s|E3P#sLLX2-m!y@7^<$38HO%{_pdDpXbd3 zbI#dkoqbt*?X~y1?6v#0yfYcv%00XP50C~U!Tg`%+fFvjq{{LrsQx~V` z?F>-yb@X0d{0x#nFqy?urQPA9#M(|gg|h2{o&JC~cf zXqw)J@|32xKeIgjZ#dd);0?V6_2K_1j&?+!gG6t>KL07bZG)lx|C8RFa7%Glg94h& zu$-R(3|PP)TEJlfyhea27nTz2GQ%uT*W%obQ zt=`GfnJzro0epwnf|N6DT&1YO_^8X6zM}m?I%hC+gVhgp}Ld_G@ z8-kJx%bgjh+bz@s7V1(#-2)UxzJlMQ^#A<6ed4u@GRD0}sJM(Rk?n!#k6A)a^o6E# zT$?{SvY03)#i1+x(Q``tLq?Q@F7rn(Df17xxGZ#$<+i~yS;;o@e+B+1r&UMdIzPZ4 zovw}FU;4Y5O)Pw)$JhMbekr-DCh~E|E9P8*O`+Eq`URasUoKaz&!iC(|CK?^>s&z0 zQ;*Y_oWZFk|26}4pM`qNLQQ5L-;xrbXmYOyXmSos?zCsx_AaCrsp?~NO5vWO0^sE!^+RgU^43Zx+(K5ryl({i2SjuTRZaLK>nW!DCLtQ#v#;)Nw7yj)j#A;lj{3gOXxx!%S~ zd#l*zr`X#PZcnhcW!x5XTgO(1#z9JBe?wz`Za7TGLJ@PQtZvA8W!}bKdQXfIqL>>q zXtaAx^qhh~^e4sP_Pk&vfq8z4?U;bEjX#o>B37GARC7cc6M%=)(M${;;k%P!6N~QI zrSNn5s6QV$*7MUWCx#>Sn@Hr{QLGhs!$WzNdTzkWi8b6RNPtUbh+Ll&-bTUHa}`Xz zop9vXYzuPR($UnLy|J}?P&cVb8|(_G)N6rBM4s7H8%gLe#=1$b)kKoUG94%#&C!hh z$Xlt{G8Hy7w7hftF%SDw6xRmVEbHJeQ*QZM8*7~tKR>N2-)kJSa-B$0{wGa*=9Cke zoNxxJshqd^=;OuaMVtJx>UB$a1;uVj2bI8f&pT}6DnM>v=D8sizuh_yotrt(v9Lbv z@kBs~db-92jncyeeO;o6BS=w=lnJGIuCU5+B14QhZEZ?$XIgk006=wgGFk5BJfUqZSY830Dp#S4RHzKurAO6y`heX1Tl1IK0TU1VRx@HfZh+JK7Q1|8i^x^Em?ab=NfKjrhO z--)m196W%>-EQ{w}5<*-9?&LKGI zaPAi$<}q@A?{Iv>58h{o>~mkq*c+aN;7w-i9nU(^{`&X|Wi}%#v!A9i>lx+ z*+dzirun^Xuzb>+Xcc%6?Pz8Qo`(?}TErJQ)y8Q9DUaX`)rC!_@=e-I+ypK5=uUam zoxs37b|31Ixw7XxN?So`WKq=f4tDhB{~!WQ?;78n+0jc;qw(z=EpjYzq7YSXn_^As zFJs?H?YG56;6LBryIS!J*wSieEN|>)8Ybec*OiuPgqKXcBD-ACvR;4d{k>#=7vGD( z2ImBi&d?8=wFu~- z^QeAos?+{sdlaA0Q$|>%!Dym7`@`tNAC@!S9k*OrFBg@NhLWTgz2q%xRxqB?SsFl`COU)eIP zNM)w!Ht{R-vJ;EiYK8MspmGAPb#fB!f={%i|7ZN~74!OW`4k%p=&X;op&QI5Ll4=} zSJ{EBjfq|CnKYJYAI)q}Y=CNPudqAOwYw9|+>~xlZ&AktCpwUUli454_y1qppUCP( z@l*}(?LO@kvrK4GPs}tEleRrg)|Dre9{e4a^TJaC0htg5S%KXYLb2HOMhe^dv<*+?))w!6oTLbdrkEZ2;l`$g^ zyvl_|_+@`U9%vriDQVu50sNH(eAogGMm@1)1%O7VOHoEF|7GqWv z+wIdr7-?&*6RoX?UZk1%&n)H;HeT^I9;LU@i|`$-ld!SEi5*iB1tpV)-eG({ccz66 zXZv;u8;2%b%dDJl$6|pOu_&vRtg*%@5LcPR*|NZJl5Q=s#uG|mJy;Ql;FV}2 zf37BNzv9zl=EeEsd2tn0YBu82uTA2`5}#hS;#0)r$?)|@tB|35wmtWdS{I#y@H9c~ zSrxDi2^^Sdh+kE2L29}VP)SXC^fpd1AI`*_R8@r`<>Nr*dT-2no|Af8ti9cdwa;mz#>N&0D&O=r9;_6y zC~+#QDP=dVBXyNu%FDjO+t`zrb(K3TR9)o;Z)2XxK(UCTRVWLpki0(Q6B&ZNjUSqi z-@-=4jh{F$jjwRy45HMkp{dh z@R2#!m`4IYM+u^3V$x9v5NT|~3w z+;$|X*Xt&I><}Z6R}kjMf$}%%BF%M?wf^mE>vGpi646q)Hu7OLVJQ8PH|xq>CEibB zU8LO~UcWajt<*)@Y9sF%YG zJ{$tJl&uE?R;TAjpl;KakxJyV5Pt2oRNEeZ&H&`zESwq*vbZnvcM5@H)ZS4})DMHO zOY1xiB^}z-GqkjNTWhYoz@6ab>}6nrFZ@eO^@W4Dr`qKtF?m>3GHwVNtUyltM6Z9x%ZErq;YBfEysHOUhAGF2)Hiqy- z^w6WY5#~9Sj~LB{x8YMb9{m`2l*3Ka1Tgiamg6a zdR0`o+#hZ&sbk+Tht|ERbucQmhGq@*MP{Vvl)6@q`D3@5Km2ibx;I1LTD1*Sv=1DU z32!I=e<$35huM5SR}FiMBC5`h4J|UHA^6~e5BPvz`{0@f=~8&Z<{sWiqr!vs0r6`e zv=Jz{K-v}FXeSWA_CZ^Lf(wjVW8ebr2QJXE7x66gGbW->ps;BLAntPMZ_kU>JS~QOLGQcd|DVE>iT0O0em9I6c$4ZN zbf^lSYV_-zyi=)%uqn1)GVKU9h-u6w-VvHR@##c9sf)y!^n|Smny_*|b1L^vJC;Kf znZLMimz%Wtwl1=wF5)KcCI=N#qC`rXbq1Tz^k!siR^JIJ*v$Ce?F&9i!+f(IrD49B zM`@U$?Z9}_TbgUe^P;7C=y=|=R1X|4vWIN+YF3eX1vc~wbZ8iXj%4UhDHmo9f2|v2 z2uktu>hjMB6P86gCyHD<+Z*5M8~9*+&G>;K@R%tzyc4}KuB&@U}!ZX(4{ntn-Po+8DNn|?`Q z&LYLooPJ4R{vyQ?o_e%(V7Rr4`hg&w~;Ry*;@4g zQ$bw3FEs}^W;RICq^&>v7s3iT&yA@0P`(yyNPN0dWF-`buVZyUpeJZqc&gFbIeQzwgp?!i-4$qF-P8h;_?uMO2p z?1+XqH)_T7Ai&+H9hrEb6Wysq@BG_LW52@sO#g~w&4Tk)B+gl{cJG6YzaBc34o^O=tKnU42C4@1Yb#s`jjG2VaN8}ZU{>%Ngbb}4>V znmwX9V7)MMu|33H?->p{(K1BJDM;IxiN?>kWL6v-)=WW^sDonz2FYW{Q-uf_uX+ zbgCaP@C#B5ztA~;q`)soG5msirNA#pG5kVj_>ls?AjR+to#00b{DKt2FSu82Nk~`L zPFt2=B#xt^nL(>D!wA7%@D1~OV62(nYsQ-Sedt&-zYiQ+xZF(rW>ugG%}o7fRlt;% zp8Efo=`zE28}E{j<-Iy8iTE5~{VS@HU)`HlmE1MUs7fB#%09ONoywqknSX1t&$k|BPNQHx zkDd)jomgBK*}%e^dHV+WhQ7i5_si^=D;Rvu&T4{JkC(>a6;>q|m+8dkMs1&VqSxbr zIsk#<)a&zd&K)vztGCf1J)k2$q$!D={1h+FpLEreNRPr(gD)u;L0mcnjiEqv4rQLiGWux(44yw^C_0T-P5KDAYQIQfe?Tg3 z3v}*0ZIfNuly`XIAwX*)q@;_yiv-NwQe3PwMJ^zLA~A7~a6+?dE5ES$JJ_+d=Sag4 z92=PAaH4fp0mj!nV|;b);`27Pves0t#piWJO?lg#)5&Cq)6E>pqKXB&-D+gNNo#rW zMj-M+SDSv~k-419@)gI1Pk)`JhK*nnOz+`BKVSVb`nmtHrk^Lyr=RQQ=vbHhe`MgF zvGC7Z_#(mgv+$=0{um2?YX<%n3;!DnUoH5PEqtEfH_b*jm(345{Ue<4#oVN6aizdG7Ic`!e6aebQZen=<{d1-lB=RjS_ zNjb&UYy?MFhT(cd+1=X-HF%>A?zxowQwQx#=N89{76fkJ?0@*97yGI!U!DCVMr`9! z9C^K_{3BE)W(oI3_#*<5H6@EO@*S2-k>E`>OvZtW{J9AND9&c>+4DH|Y-{`QP+Gxl zvQX=9VuAWQUI>=eRlYKZcvv@zNZEh=Q&An;p7#Xoz6acawJ#vaDx;&@v|h;v)9Y77 z6T^wQ^7Y`)$zm>2%PX_1>SELK>MCEK`y&%;?DaWsC+0zZX1oS))g|lA6s00|)Dh;O zo7wQHB0Tr;{j8>M2CBKwlPX}&egCe;1-1s>TeUy!-vJjUbj+Y5*YW73Sj69Ew4;li zRsufC*whb&KTpz{IyU#yFK`iUItRudol_!3RxxkiAmQRN2lw#Cmm!2e^umhhVNPt= z`#~stNY6LCjV43|%?EG=ZtnjRlFLZp&$85ahLvn_DqCSCTfv-F<8cmR;|FG353l0e zU5FlN3`wB24kfQLDoa@hIHw6k>*0iIx|&j*As*a6{$$dp8%GtY^Mzha8v08q2U9B7 zLhN1Jw=>lT4^GaN;#k1|r<0d_V-!raa&zb>+@dA%i;G<9SpQJJjn7MOzDu<@NChs{ zrUSm2xob)bgB$UL48bGb;sTHGqh?>Sr+?=(_1g^KeHQRB3wRoH$CB*;nrZ6B4Ak`&>Sha7OL*2L z-jh?))I}Mn3oX>;7HWu~e1g(6b!rCcWD9ksg_?g0P`fFSY3czui+k(!u$PQ%uJN`~ z+$L-Nr2|5vJubeR7Dh%^AqVF73z@>@bF51;`>AvmR_c`$}1!c9j3x6%PH7M%tw4UQr@!qEnXn=#> zOQUc`V|_(`p9=~GY{ICl5#~#Nrz`$yA|W)UaXe+N#Tn=+PpQ7I<$Lc_O_AoNgva~T z7XPHe#~v>zThlUe@|T-AXs0xH`|DNPRuzWM4n)?Y@bgDr&G1k1C@jT>b!c_-M+b!8 zJFEJ@7udkXntkerfyzChMeM2X(tmtQIzD3JFK7}9>JYFOL&5r+>u1b1C{|4%vd3xO zljnr@E_;A6qkf;oo}M1mukI4H`!WESsZSFt$>Oz5bK0>XB<%`-K3O1_MoD)iLFX6Z<( zxjvBkSotHWToky+sv2l(02Mhwq+1?fsFKqylZ$#roe< z-RikV)u8=m|D4+Yu@m8Z8QKr?&{h*+t-y(_b63p}!GfMEXzAl&{*J3F}bdD$532){>!50c^jn@ z6_r*vi!Z*&{N=+#z2P>^?wqBSb`e7YZ&;8gCoYd18{lg4$NJbk)fB+sJ2%$51Qavo z2q+%e;C@%tV~1afewdkprDe$z3_tK^dS`e6?t8poM_gUyc5g!^1(@M{ZnSqhy9@(V zY_PXc1sYNMyjbB}NU#l`iM9el7q$6_2N*q}uKYu1F{WnyLBDZt!(K8Xgp$9xqZ@jA z+n0Xt@Kz378=UD&Z>lc?L~pTuNdP1WpZ+vWbT6E)CdAKet=Xd{(~q0x9Af&2)yyaE zosjtXu~~aq1+^9si$)~;vAfr_yI&x3V*w_vFF~&6W*VsoF@sYrZ9JT+CUlN1^0O1| z=8x2p%ZP%4@Qs`X9lMnMQU?Q(X*4vL-RS52(eL`pzuc$Md&sUXveDnXyBkb`s4aQZ z^3BNQ$fXizsj$PJ7r$7h`Wck@Vjy?Dv)Cqgc)>xtD3zWn2!=p{X|#L3X-{05e&W7# zJ2z7Ep+Q71>Ex6sR7A|U<>CC;yOsolMdMB~4M1G~xB;iX|uz zE#*}2aqyNHk^Ta)S)~(_MVjAWD)G8lX$w1kL)T|=sZuVZ_!!59p{03KPgq{Sy?YkT z2}B_3W)6~}fr^KP&Cbye;@r9vdy^svG_;%i(W(^)CloMaYqj@Ikcl^=gBxFi`{J}+ z9f=!2z;aDy?s^_&gJRqlnCWanyxoU&k4ddcKMs8mAT(3NDMP5%;LTdA!wo*005^zU zXE?+Ax}R(BU%rR?X}XWBOdg&*K1mC3*G^R<-*r^3P~!fpAU7wOZ&?WmKKJAUb0Ok& zZZAf<8NXTlBfOD@zhe1E^Z-LT_@EcazTF`4tquZduZ|uPyI@avXP!5_jx?6EjNKc3 zoX5nIzq~~#$1qaV-JhCLv+26UyMUDooZw@(WvVC%KWa|(#^Ru}n+PeTNp(7RI# z{N>w%2b(EotGgYtt;_wgj!7hWM!Lvg%~#9X=lg@0z$07T2SHU`WJZA#9$ruk19q~} ztX@V}I+}vJyh(f?AzJLND!e0)g)dA9(FxO&{6wP%qQI|mqP?3r4uL@vEFGM*{%O$P zD!AH8$U}@`aOgJEs6ey?3p_gl9*5Fxg=q?7GE?_J>UWP5Z$a0Iw;Z#A7x zxI=g}CX{yZR64&k5HU<_`4rJJazJ(X%)DuR{gI7Cp9>G6uUhGyL_Ivk-~2HQg|9Sw zrXZ)j?$@HBvvd{>Zy5fw*WbJsrn@)8bXO6Ct0J4}KF^#9iF6D#^yG_qIQZo0V$vv!N z-C*q)pC&M_AE@~k} zw}Ow71<3=Fp7e4hRoP&^+j5OCvCe&jao$DVfKi@aeM$Aj7dnKQjLyJ$S3>*VkT-dp z(FcTA6|094aL*J|{mi|AoRTiCmD7qdOUoaV7KolMR-kVW(_+NpkM1zvdo$mkVZH}C z^L3f6+ojdxt1n9&&BNEp2RMdoXScumldu{Sudbgvr(;VP}UE=QJ)U0o=Z4Qw%y z>>bHPYOVIfcg*n@uuUZpd5`rg?6m<1c+RDDvCA7}GOG5;{9Nmqetxx>o*05Z>cN;O z4w`c3$F01|tZ+VcpA;nvP>U0;zUoMAR{o*iCr+RM)-Tm$65nx+zbEeC)gFrX-Xwc8 zi$Fva^qo%Z+%KY2cSoker>B15L{02BuM1Re_BNcQ_C;3GvUAcc8yy{P+Ql-$HY@UWp!}1dR}m<~t*k;nb^pb1 z$h0QiJSUTF=+B9dL2NetMb9auc_Y<4Dbe2Y-t<2j9+9_G_OJh-eXMxsHW|38OxQ|q z!+Z+y#~PcsXnr6@bo}dMrTGRQZNFjs3Vw=E{}FnssW4FaV(7yF8&ym&nsQ~z zN#mMg`Ep#B`tI~qJwkPz#U>M~SM^9s^VIP;#1>U8=0#OY=0%Gh*Jd8q>hYhM$A9Wk zHa7dMtU6UPq}fNAYivEqm?kz&V2RissyF*7v-kzUPg7k23@&;||3Nr$w16!HRUUO- zJ2;ts0$rCHKNo)yowGYKhyI;I|DFlR2>SQ+86QRtqHE9gaIke@)G5^WC_zrC@=X7A zc6qU>Uqo-%9l3$;Z{Yhg0MYo{%J`cmlH=G(CpM>GX(8m{Y~SErgpRM~0Q?<@>h7IC zj=D8qXBf>DwHVs3_V}J_^M5=RHP4=Fd(7dgNck^g1{`~gdkKNVw(cpL^^qP=zMDtd zBJ$~+&vaN|)#vedU)?6is$KEjBif7&m!u9J)=r!{yZjE%Fs-+Z2My1C?<}rzz00`#iZFsuF5ruTS**fzUGLO=Q$@j&fb>x&_d3N@HXDcL?~J9F!}ed zK;h*OCNLL&d$uHZUCm|mS6$Y-gZ+``BCo;ZnTy={eD9B};w18F@5qgH(VrC5RDM_! zS?>*4#_!zaZES%u;r6`H;-o@sscd#15h~2T5oZ$5<>+@B8JC~AkAKdRt3e;2z;8eo z$eG*&FF!Aa)kc%VtJ#ypr|#%2nj&^HMSNn$wH63W5$Jf4x-~^q|APT$<_JYh$mrM7 z{B&oLU%F;E8XTDULi^{b4wKTjedY_xRUsePiKi45fYEnk%@vXV(Of~KGdoojXsYPU zI`mMYn7q^cJlc(clA0|HJN;>zo!a^M-}BSmx!#5!l4E+R(DAQd)4tdUCKJ@cuD79| zzKE=%@XUNs!}?qkMuA__i>H*O|Gx9ZU#>Lsg=tc{iOC1E%hUX@)0dhb0+B7=nG`_#p{4gzG<^~BQ$S(4dMCOvR z%In^811qWW%}#DOHE0HB2}hcb_T_KEC-&uS-+{Mf^p{TT|6Ts3rDOHz3rE5B^4l-S z%b6BJ1!8ntyI|Bcr5^VT4w7)6M4dxFvRfGO0}}%B_a#kB#dPeBt<3mwD%*l*qJfF; z9OF4$ex~6Sh2-5l^&_1t`<_2?ehN{O$T3ikx;Bj{vsaoG6?Ed!AV>(T$U#LmOlr^v>@| z$>^@XkC^q3!Tapyw)2cpJWp4#sKTfo6P1 z-ct1v)_|92X*ZT%@&wwJ_@imr>ke-CatLa|;d^&kj|up=b6tyJXx!5telLHzl;5kK zuHm=l>C1Wd`NXZEvbkHid)3^%!rdNow>v(otZCv@-p1q4jeCa!_+E{-P~~UFxm0`H zGTEF)FT?0mbosI>ey>`_imMf)-`-HvsoWa;uDrC&Mov#tmQ9Ol2mVI4jcZm9;Sdp< zLjc#ZW(TDXjbCpji&`xx)iw{O23-nEQ^QOb73V)&mzV8t8Q(oRpt_7Ubs0TQbs2s6 zvI*44q^Ig=(y3{tj`&h+4fZubk*`;dX@0HxJ<0xo(QVX^JF^98EBoVED3^cYZTJ)T z7jJSRpBnmr4;=u7aV+nu(C1@e+EtX6>3Sy|jhV?jx7(>KO12PkY!S>)y)~s~YLTj*u>F4v@(n zq$DYp**3u{kNuz(Ss+qCl44ATlNP;nSKzB|~=7cM)4c9&Bhu(l!(VTy56`MZ>dWe&U1ivkgP?o;>G z@#$KMcx2det05-)<$1CGfA_~c_hAwBM_Lnm5dN_4bGAx_u~H^uKbHWBF{=^mVrG(_39|v}WZC8hcVaK`0X?V{S3zC* zmqGSC4Mo_&Q2UAd{5TlK5wd8c5#PAL^fn#3?W@SBQ!zTENJCxK>FM4mtLb8^E8e@eNQf%%-g1M+SoDWbJBr#^9om3EX;t+zq(U5saIDGk zwKsz+S)Em0^9GE;7_@W(7*S**ad!zM%ytM#&JpsH*#U;f>aW$uEKJ5`Kh9Mj=A0i5 zerVno7}obV%%kvE4Cl7#Xx8+T7^Rv3_E$cUWYgt4kt0*%?1wq#R0d%^fWoX{ z`%Wk+T1Ue_OCEzfOicsP5WO-SvmaYq(Ov`flAt2kC@?ZoZWjJ-R&}PAqhMyam)2c_ z^H1I58T1Bsf9q1eX!i{M(5Fnljh`;>_lv*86=1-&Al6tn^77~SrO0xoi@r*KC4GI@o(vEa zEB-#5CpK(~?hEbxWr#j*4)>C`3z9`zlAUgTlKHyy*Yd23v(Cy3$%GVN%6-S@{#7aP zIv8e(h2BSs)^Q6iku_dDkz-9`;f=odj=5678P(*+lbOk9mT}NU#OmOsde@Q+QCrfr zc5fu!=j)XA*RzlImu-SZ``ZMFw7;@8WNCjV54XZ=J=ad{?by0>U$eLGxgrb!Y~9Cfw3@ntc5|ty|ceXO8?H@pgAkc-k^Xh z8GaGRD)4{P-k?eOP)!f-{}@`yR@C{7{gPL5dCb(ebNvsQK_z*Jz6_-N=g!Q_2_4{# zZWYxip5CjLw#K|wF)3T!N z%WY2cr^uhH{G3Ip9pRTWC68W&Y_|q|&3q4tFJd`}T#Pq?oC4^kxg(-I=c6ogBKUH= zXXe-1%A(*EwUrlmg0=h>1V{2)7#!(`H#i;7`@<_YZeE#sRZoy0W%?pm_ zH#c-YUx$9p?)T32^Pz`W@)>;aT!7pbZpLT@I;5S}%n_I`+f1`}zAWVNtz*|D&f!Lt zc~d#)=Ec;Fw^7HFC7aOjm+}#2Y?;k=i8T@q83y!&*WD1UUK{Bjp5fW38 zv{r-;xj{dw?loB^bkU^Wgs+> zT$Ynlf0jwAU{&hGSc4{yOzYPp%^lTR^Vd~=&zsO}=%nQAF5|yOaND_Z z&b~A5=x+xPPnKuqA9ExgCWyB~y{ap>hnf;Pk}V6L<^)yuqdHI_pPQVV;Zd|c5vQQF zN*1}CYO97w%K_+)-2+e0%}IF3MpLO%r`8av+>SBc0Up*3BCG*w%)g#XW;)^u= zl6-MTy!5A@!g(ZSV2M&jfs zQe0FQJxh!9d;OE!g{j!w)H(#-f*a2qH}nd<_zHo)L?ad$F=zN$9K-ZlefI5CtGe zFInSfz&7# ze^}6t?mAhsa_bBl@KVMHx4Se4!_dVzxK~RD6y)`+(wQ82D9vp1#~$|$zj1@(97R=p zX7czhA5X<1vnK2NP!rBw{^k!o*o*s*j1296USw8(Cge?XdSa6aC|kt@aNmPik?Yhx z_=OHuR%gp`xz(+uM5*p<%jG(&nh|f*BR66AFj9aelEfF1l|dB3G+T<)ePZH1_jWOQ z>+Q%$k-fd4yCIQ_!0jrhoSk;7Haqk7c9606CT$jkHsCC)BHP0sd#e}wNC|jrTLa}( zc)ReR17znlH<2zu{51UDjTk=fd%C&4xL3n>VOo2yy<2`nTz^#0&y`L54A44s&e&b% z@+0gCJYCsgF4yHw-1=cz)ms^7fzd~i^6pnsUi(VQ^ZTSwHSK6otGtaNsvyaAk)w^6 z1E;svJkIc{hdHn=sca6pT9S>D4hN1j*`J&8Ip`$ImUBdw)^`ua@}zl`21KJGQ2BZ2 zRB<&}^i1mm2AHb7tnlXl!CAEp5Wgzm&gj(L5EDK+{ z;O`}`$bKwKUYz}SWb&NUqsCImL{#wU4hqnOz7}saRnTRKxu1{7YS6l5kL--ry07W2 z`RLcFkN(C-$jghNqU`-aPV_d*SLRv160v-x8H;AOD_2{-GN;g)^a46XSc|7^<$$KI zU9S`qDcgx2N)T+OpBE{}s3&rrA=NzJ&$C(Mu5-6PE!H-oz~Aoxr@6hGe_6IdP})En zo&?+KGPomOeUS8J@E_-l!smlm_}R~vi=u)7xEGn+eSkda6O^*7g4&C^8U2OB{s6n) zDaRL2D)mOhut&O@lt-~#37h8csaM%1O?TmT$S;iT2 zZgGfi>h#zr`T&BC)WS&>SxS6_6|ad^ub0Gbv(<=T!x<3Ez;Eo zh@lZ)Bh1n%m5C63f$X11Z+e2E8>_gtVGl}Re?(jzYj#(zUPxWA5bl`K4b6y-#rB@P z-RMjQmIe)3rS}0HP2EwN;{l?rKt_1*9B)+TkHV6T*;_8n;2KT$CvJeg$mmO#(ymRHB?NQ)LKpCe|XKQ=FB!m(P1Ev3FJ?euPadorrg?g@EyLka-a>xyrI( zQ+6o@gs->lDJm=#+C2BN-sP?PHUk0R!qWKGF>Sml^4yz>|NhE=cgUfzi+(Eo>G)=M z9z4x_*yW>tB{}kFn~ZWVNRd{ewPmr>ytA9LOx#O<>`^6}FOy~Gre+i)m-tkSysV2?%$mNsek?H z{(Zyt@Ac|m;{}%P-<^S2wffhe>ED2N1fdHqQ2+Wf{p-Kc#-xe&b^4@x#Sq4er9`(_WI+nKkfYk{dRD*9D-kJlenD`#w4b~T+;t11{ zYVYmM{>lz6oCwxj<-MJ#qr14Y4p?2=z`pEizO5?R>uns&P2|(?M}0&WEd3uo+QVNJH@V+PJEb~EPme7JWHW)_zf;E(Ph^1s+MYkwk)5ZKu%BdGy3Tg ze#bps#qZ@$Patf|^DPsve(-s%pQ@4*r{chwG|Q*c@U;x|C2-m7BRK^(?UG3fQC?oq zCog1y9od4??uAY#&0yanSK$0||3C@&oHeZw)zZiKH>zcp7&+z7!?>O8_}Zg*{jghbD6p*H4}qCy1~M8$&NP+RD21`mEiH<2y44d5%6{mKODsku$#DiAHqesz!;$l(>Z z7PWi!Jw z)B82L=%cGh2LW_>Nzn#Jx|qAU=LV{^EjPr9rx@UMLV zEs}IZ411}7tkUt{WE%gZFX|7#d%Bx0Fp`sXgq!dJ1n|7KK@O|p6BTuAdz!RKi>PJP zfj<90b3qsVkI`jzw6TGD+B82Re~P6{-11SujfAIbF_q&WQXZd+?}RHvX}u}D#BWS7 zHD&mJjrWwfQ_TyUM$~&a$G2>_a!H%{+;pcL0~9_$$!l;hz@cDcXr8jHi+rJFLPb&( zk-u^b1KiGAoMZVAzV2ob#@J=5cZaEv_Od`}3ECTr1mDCh!QxAL941JW(9@lKreRz? z+RI4VIu0=E!`Y4Fn@~(rKldC1(0{FQ!LzvYyrQuq1Mja!iKH@)jFELow72pzJWkvw z?%u$A<21<@8q8Udt&aS}_j+bsj-PXEf;y)YA>#2Z*-o%s+tY=<$KXYSvDH0?93)ixgvgKF#m~wd zi4`}qfPP1_MdLdnT6}^})l+(J{nM5Dw%HGbSi4y;EAPa@;jTJt~hQIrbP0?V44--c)>dAV*>Lix!Hc6TI z*097!t+2*@4yrXtnzT_U{fom-++Tht7kPyk?03mEKiYqu`v)mx5!HPuEy}Z{=HothAgu6g0Rz(C&EsXgZMBu>~!6)<36J=EOQqj zk|w_u4OMPGx#Nul*g*!AvWNt^l)OtH5nHLJePaWC6w%)u3lJZSY)5i{~Lx z=~KZR5{veT0Zd(8Tj5&JhA<03MC6>w4wDPFYIQDXX<)}I79Uqnvnntmq!B_`1{buRE z*!hOdkuiCL-H@hjlPjkGL}g0~Jliwle{?UAj{9qLUuN&8>VBZTpRW5Vdw+xO1Ll6Z zw!1l0d9>2Tm^66)jD@JqlVP*}MAB)hr0KuV<~Tju(x%_evu5>FiPLZ8xm3@#)af_# ztX@r(JpCq~hw9muK4X5e!A$S_<-hYbKFH9OIfqTad)b}&7ME~G07{y$RYoZ#VU z`%S}xsIBN6W57ZB&yV6u>O|QJIw;c5DDHl7p;6p@pY`986$cr`UB~Gt?mjtH^8h}u z{^+g$PBr?LT!wxFXQJQD!Dqr+AG#L$_4R4>+5ktshfv9~ZU-oGO1ag?G`?0fY;4YAdhsc<)@1Ojbuj z1>wEIl;GO@B-W$z+`JPU1S7W}qUP#2Gw{Nrt-1a*C zjlCSGgi7!w>u-!}uCZ9vldHGk7Udlq`WlxUx6IVcS`p5V^#5F@Czjglq`ss^pSYKt zVi!xqpcweM0wCB$(^@5KzoG~feMv)UJ=rl8VBli)qPnAH3Q94IZ=O5vY4{##C4B8| zcpo#Adn-94-r{b+JE?-_E=c|=d;OvL9m2bwiGD1)H^|MnMZ;N&sv+fSSWn8ayptIH zmFw`!rYg8(YxXNn9-5JF6)(VF`7b%qrs548L>-cDTAo!AQ^-8WX{|CUdRey_fgSFK zlZ6E0gGjh@WTP3bP&i|fy}HylWxv5aEoHy?6W?X$PY#F%p`MAKp}<~bvfvq)5=Dk! zeRu=#i4YH|@k9NogSSy$eF?3G3`%gSEXM9p5}fTWIh%_0gQ4ifWG8o?mI;|H}y65wMTLP zp6;Xhw`cNMWb^rr&F2qXmON`2-vP>Ji{M!tf2T|Q2vLLSBv#>b`zv?W=x(nSSxuO% zWH2+TBKtg1XWquiEEq8XxtCP$>R3LQ2lt17qP=GqQvL=Uf8)E0a$2iOs%Si9lmFQ{ zB()b+m9|z51e$x}s#1lyy=nr_dRZ?_t*T$FM<-Ta44jkv?c7wiE+|j}elX@sIm!T6VbU0nPc@Xj$Nj>Z(rW{; z-CK&LtJIajf2Ad2Y zUPJ{Pwo91oV~>>HPxi819BsUpx_h5Gcws@3tpsU~HGvM(1i;u!*^?<$Rd>itWv%k$ z=F~>7@YF^_g<3Dvq7y-GnWUd9lvA9)dR^%9sOLAmsC5H9ndC9cuAlQ=pdx0zPOdTi~bdJa0qtd0ilX8$Cn zHe1>CbglW8PNxqC9U>ULbXR4)VM-ZvVCKyuS#MZf5AtN*+@AG@iVW&V7v;{(def6P zJ2P*_XT3p&9`t4AO-jc!#N11z5Db- zlgIMDTZpK|=6&}Whb+-j#ruFcHXlWb8)GtUOY9Tf^U-`#v7FB>)m zd%71-vzTSsP103%wqI zPig|8?Gs&-8QLfu;`pv#vtzuE)FV;}7-7ykjR8uk;fA zUsjEvQGaSVsMd+4kD{uYe8zBHcxUMZJpy8`S}4bN_$*L-heE*iWJ*`1Vl2nB0YQ$c z_p+|3hR1uVX+)t&Bt5E#lk-ezJ(V)vpW7&IGY_@EqSHBVvJ-14y`Kct!_*Z{SOqnK zn^fglrB*Rx9!oD}fM`V(waR~tz1*s~a-ecK&eCb8%(&vppG*!qkx}@?++roJ)Ev3W zQvej}iXG+I0+J0t_>U8mztS-Pnh=C+2}+&76!AqSBWf!gVQLCD%Vwx_z-srR7SPIv zgSCKW{V1K~1kJxA@vna7rk6 z3aY(Sr!M9p)g29`f1q+ssv$kNry+mlvfqYinsob7NC4_76G48Pu35@bRbAtx+Z2GP z?k8)y=r0;tskQ|w)oJ&TLg?JmQ5q!t$dQrN9Nu-va?aAqTi+k7e@wb;sQhFa{(PUx z3lpt$9t)Ucujz-K7aL8Kxme*#(nqivB~Uts#XcdR=(p-;swmjR$b_TFA5_f5;X0Tr zv4oK~8aS$y8WXoY+~Vl$_68kGABbL^v(%@L@!g9K`pr-{S**w4qcsEDFJNNps6zE( zL<2z$*oMP&tjAOAS9hO+;Dkg&ig>sh>W0g$3HC{QNjgz>y4eHdx}1-;!TgdIxTvSDLeqOm4lY8B}9+)O+8i( zk`T)9K8xwW_A$sW5s{9Pcvr6D;X=La`d8k?S@x{-YIvvn{XQxc6}WPsv*8X}|0>CG zmdGB-UlUczWvg9#FYisIoSwLpiOrxiHt6T^G4xL(tQ_Zdj`;JNb76>~e!>_s^tjr*+ zr$yM&UxTmXkNuDEm5aKv{V?xMd5qSj!lLP~L37RnTels(e{7k7J5~92VhiF4ly+h( zGRmNDAA}vU+#;+k3tb3=s|ZHF0sNGHBlKpg)CzLuhi$ikO{SmK^~_6#osrZihJj3w-^8nEzS$I+SjM#O0W@Z-KavHIAv+K8QQcAg+f+Tu*Bc>EvH|45qK8 zul?bx3?)t$zTT<`Z`ENAb*w1s%1m!z|uOATw2nDz4@jdgGhpf6Y1(b zGkj|4|Fo&0j*V)Nvra!1E42Ij(ET5>upn^(iL&>T;Y`9~K$(OOabv3qqI#Jzav7}H z#t&#R`lx`nrXAXM*sIvC3c|#442pJ`(cvy?CK2tFUgiXkkeWN*Aqnjn1J`k9>a*cf z(z?MqvRVkRAGGV*MVWYk&ngoWMHsI}OLx|

%x;`c1AW^`p0Ob`sOvE&O06nTQ6%AENX|3b$LApFKoO56|I2!-lD< zmJLzA)1V5eM$^hDn9fLM8y)j8=p)7*!+e1*q%R;n(F#UvmNPS1PVAcHI^y;0ljUwI zVzRu$W;rmGW{$mYoiOec-O7%Bst)k{m3${ZsiIg?FIK>g%dT?Q>Qz`b^y+AYT| zwVm|`lH39sXed1KTg9D;4eO)S+o`B3w}~{><m-jIkZF>}H|>MJ-zmDO!rrgFckD&(*^zDt-Ng(Xkkker$R){?Um zOX&)YdbF5q(CeF8hkS|MbcO{tDI(~@MM$w(rBmG9^ek34Vit?!5^`=#9-TZSS%{D4 zJ{3*enaRk0+m!iMD#*kLeS3DYDtS@z!sHpr!L<0iJpAD4!Q^4vD9^K>bf~~^?@Ywed;=jYioE4zv&WYrrz9@x-L##-{eYkf^d6Bh+}=kEH8i_ z1F^qVkTv>IY5Owh&(^1XE&f{!4!ib@#eYl5*WtgVEQS9g_r-spg&hraz^$@22+8UJ zDVi&msu!R%)%lnMcXia@`fzs5!iUSKc>V0*n9+oI?^Cm@>n5GUVsj*G&2x+;Yeq4K zEVI(|Pa4^q1!h}d633f8x&v)?sauWqnzYtH3`))^p+`SdHhtvERlw{WMs@VolJ#I!Rs zM?=!DuI7~{kVtsFY7`sPjj}cR%hcL+em;y1+FmKEM2EYUMCcc44=w+Vdbz76V~=-+Hs!)oR+;my$N=wSEM zH>qeYTO+p1R64+ZG=-0>BsT#Bc7A!bz8nUO4#=X6JZ=f~?Vq=*Ud23+mknGeU|%Qj zYrkc}xsH9r9K16M?pMFu4Etd3fUTJpy4aSsyM;7HS5J4_iT#y_*>EQJ;KZf_w3|Fj z-=p(9ry%g%Q!jxpcmSJcrd+ChXHgQ|y`Yi3oJ6|z<8KHPZfGPDgDH3 zGVD0M)WG2$Uq63jeP(&orcyMGRcOM)y{(4UvBPqe z*rG&;vNtu_U;TRm)LH+?uJm_9Z3|py(ko76t%)Sryq=mN9W>=*HkSJn2$EM4>V>KU zNGhQxa`rj;B}LxYHrBVZ7r02k_1yukU}oWaS#lTOW^H}%w`|z8ttlHa=O7uVHMa=T zal-HIcADQiR1qt3a(uiSMRukZW1Pizl=^SiKbuSW3g@LyoZJ-zCFh$Bd;{{Tl-~p` zIwkgF4j=a6oc-b4{R{RH-oDYK%b+F$)a%L=%km1^pl~Ni_=e{mml%v{^{j2PV)kw*jB&N z!1ekQ?96>lwTlhAScNXh>stM4CElq-s=twOkWDX;yVEIE(<_j#CFuZuqPy9vQ^K!8 zLdoK+vqWM+%1B;aS%+Y9L^HbGAf6ZGY1(cf7wO@=oALa&zxOW zc(~qZy%^oWrfOL(jD%{yg^| z7HyNJ5FKvsVcu}GnDyIj9o=(kybo;lhG+4pwg|T@bz}KALvv?GJ&#d=LJjy)jjQJ# zH7})zBE1R~P-rMFz zaKKRb?E>{~H2>^lG;wJo!FGbY;5}qr;8v<&>Zi<{=ol8`Fe4{A*;kEmj-<+D@ERSF z8yj>Df?yyvY$+GMP7dtCzn91$4WTmuQPWQu)dNgPytz7;tye2?E=mSVmq3yh z3@g9eYB+3rN3gk?DB2b#^v~t>z_>{|lkHA%8MXHmRZY-Om3~V2$%$|IY&9XQZn18o ztdvS9BRtl*r@J^#s0N~Xds6MUREP>^T`K%(qccUx1W4yKUv@PlUXX1 zxqeiHQ2Te5KE`858|CQCEz`^$lTwie8;O6JIYOh?P9L(5-_SI57tksq!Sjr4O@%I5n0#~}9hTCtS4pBhus6Q;F9g5vn){LYCG#pqk`+-vAvCz68UNH>K; zZ1)@Q$3ALBS{{vMi|C=fJ3E$W!eV{LNccz#&ej!(ejV<);vNtRLFfq&hM&YvXF#*R z-)Jft-df-sxW;?u%E$2NyTuIU`s)wO2`P>ix$I%B=N?U&H2LRECFs7Hyz6)6GJwsF zTW=%tM^5Fq(n4>;6PdDG_0@+v%~uVtkZi_=&9R$Sz%RT1D=wfx)UciRLSz;F5^mat zPa_&eN^EPTaCQwel^-TfcMV zJFFX8ES9{#+p&{vwjU%e!|_62u_HqGEMMVmxPS#Fo5(z9`kpB6MzpWdB-9Uv2$9rv zoM2(3HFgfI^LTH%k&+X?;0ic1ID}HFts6tmP@JV|3llk zz(-YG`#*sMf(9oj)KpQ@8kHz2v`~qNCJ^9^On@R*tf&;IVnyM~1hDdS5@C8Ap+$?W zwrcCGTD?B%qxeoFApsQudHUi}l{1b!RR}2L|NZTKW-^Ji|J(oH<^yxi+3&sf+H0-7 z_S$Q=1dArB0gEQ({}wFEDcFKVM`-5G$nu-KRG>nKoNRhEpXS2Av=JR#WSYL7iJ_*4 zudr=P4rwD-bN6}RL|{^R8@VHmx#3xn#*Tri9mv|R0m+QSjrP&bf(iF%76HgNk$kr9 zX-?v18&B|$n}d5LuKZ?h!HHyNrIJnmQL}v2YUCyzZ$~b0`!wF%JlDyJHss(-XC3Au ztlP&P1*-G8oN-w;bgCEnendL#-YwUh zJ43Dsi$taA8y+_0pfmiv)XDcw_|_I?j#cznQ7@I7z?X3XRtEBZ(FJ~mTI;zE(&ADCtaD)&%Li@g3m zL%cg#mGBm{$f5Cjl1)+2G=c9=x3r6+f^-tRGmjuowYvmHPj650|e^N*eNSj z28H%|PsYhRIfFm8CI52jwSY0r!f=tD{AHlYeZI}z$LK1aHJu8z<;bGip69f!G`7Et zTTcqc35#wi{<0qeRiN^JBmhW|VcZnk?+(LdOGSrqCRj#ceq{w5J{;{%G$wPS75mx< zegw;3G#+XSwSCrfPENVh1i>XZe!VkZ4*4LMVB5>q0y&oZf>{FvGjVv-;;<^7ksqZZ8f;LB+qywAi(is!daA)wB&&|rDg-{;<+>Bvnx-rihJ#{L5XQB? zD@d|LcH1|Nz`J|`{sNWY=;b%>FWv{~;I_hrcg zeV!^gK6RMyN;AvqI3dbLjj=8*`vBg_o1o@I6Zk0!RBqv&BbjWcCA2b_*K`j7f!m(o z7n^UbMe#kB#br8gP343{C>zL_gvDEXJ;8}0>T3fey z_ebgSPHoMsysUM#HM1v&))nYu)VdOVjaes25jX!_sSsK(S#i|*65hwGA4~L{LS>mV zL0R-r79L~G{RP~`^=Pp8?eMAck73l($NIi^QOM;8Be{yC;uF2daRrx(?+~7RpF`}j zgk41QlPD|FCHT?BIszL+KQw51KA?0Eh|v(WLXIfLcQf}mbm`P$`4m@J^C{xfqRRYmg>Q=FL znoY$1q3#l)mZNLixmW44HwbT0DD5)YMQmdiR*{GMg-C!dtS&Xn)f>%np8B&w`;U}Q zMyBz(n(;}}sYE8{HwAe?=Nb2keVMYo>>hHyPRhjQ&v zTqjdk|0HLwC|;oQ4cLST(8S~v*1+HWxrUJz5FUS=6wT*TEq0hg#oK$WkbheTX(?hZ!-^&z!m= za~zh!EP;rE;xc50puSwFf|4}#gySqt?WeMQnwq4g!N`RlGOMmep3@j9TQHjSp>^{Y zsIqrE_=}durYaHZ$8K|}j(#2R8o+)tS)1Of*e#@)A=E7xw|8UCt)(4g!tu#*Mq>6& zI0$ZNHsR?1PNA{g7`ICe^RboQXm)U`K%jEHUTO|P$Wzzt4ZuSbmm;$I@7&-NuMb3G z>UwO{TgD9wQC+J?B$>v_1$+{Qk$mE${j^+B_!aa5C*G<|}Q1{ll z@Sc*ox^*-6jAg7%^0aGoj?7W=!|Ss4;Os3#@}28Ok^JLzWA;Gbmvd@hU&Zf;swQZq zo_LBYuE}JrTvK;&O2aO_)NVUC8z*}l+g|}kP3Je9@InQ08M>Rp{Q1pkc@HDg))#?7^_VM1)k@?M$~{v*lb{S{JU#M=1#pPQ;#DZj${)Z3e| z%7MRQrwPyb>oMXvIj^3bR=|ZS;CdB+AeRxpmjdv?7>r%GUwuPa+qsT#kZWe)L`zQI zrzxT0ukgqpscUrR9hTg2b2rgqY06`vwy@Aufryw{Q1@6n#mlDNDniQD?XpiZ$4Kyp zPCxk&ONHl)jC~^No<}g$>ALo9G399z>o+@xXEIeKE}Ll6Fy=e-Hj8aU|G*!^F32Lp z7`w@Ps<)u);sL~n3nx$n>yqG8tOOo7nn$S`@MKG$>@^Eooi ztWH_`CAEb1Bg4$^zbi~i%XWZT+Iln9` zJf-fd<2#Nx2C8(B!+i&@klF}cS?xb=$*#dfr~eIXLO8Kg=IRms&|-8=9Gr~x|CJ3h zZWHoY_y1MFx@d|nxi{uGv9eQG7yTkHkQwxA@`RBj(bUW@%#}0B9wX1ac@TM z6_d)`%Sm3x9e}gbpOy%G0$V6^Q^+0Zgfdr$+)GA;G8<5AgxmpWrG7aUpAj7Hxf!R~ zAjKt9f|;v=xNvj_%-|8Y69h8oF_xBo-A%0zqdJfEdEUG;KeRfuLRpB)8X5Hv-+RpS zE+goN5Cc5Ez17~KrrMep&sxQV7jT&B%jA&H9)3Ng#1y#&5In4e+ul8^u9*&?5+&kJ zQ^HXqDon)ovua!C7-J$nJ*&QT#6VNWr)SyKPwUvz{n)a?iLIlKNo5pdTgQI3Lrqt) z>T>JYRjF9#D0!bs#R`0_^Zq6kD!wAkY-yNTd??R*d&?nxv>AgtrxM0Mq zaR+pZZOldUL_PtlfOotI*@xV}q@v6=#0%Tya<9u{`NsS%ms#|}SQ-_lfygM0IHH4X>3BJ|@o1{)M>B#1!z0;ERBiO>C*C&(}MN2!Uz^gL*cYW?B zVG=0T_UMjlzX6m-DZv1x#B`y{wzd{1+X#>Uik8#64-{s4_a2Tn=$+TW^zJ$;cwrxW zKHeYXGVPFX@RyHGQw+nFifX?$zG8irr;U+q&f%Ybqd6xfoAb1(v&%@oIqJXdsMD(S zdu-`Fr498lz?OdhH%osRbT*y6)Rz858{6402v7XLmhXLJTOaGQl<{Zl(YswCZl7*8 zPqm+?+Dwh?8hskrytdWA=1q2{FkW5I5!l?upeG7};Jv9-zq6_SVpEM%s!xF+!yW+d zy+f)C_c15Jya456e8yK%O42mUqPP?D*L%h+Z5l(5gIno!xwJYOZtoH1TRF>$RJi^z8u%8Let&E) z9PILp9y0fn&x&Gbls5q)vXl}t;usLIR5fOw^ymG5r1fV({H2ya(GbPQrG=#f=9*M^ zqQEbZW{du{M2;)KAZ(y~@()N&sk#I{!NsnoA!*&M>?Bbtm?}Hj^U8J#mvdAI62iW_RZDRdn?#lPJOO*G4Kz6tXWEa%cl^Q_JShQ_3)WPz8i+;Iskn>0g#asY zeyc3#QKT)Wakx@LsszC~MP}8?cy4fFf-d#`!zE>1hyBG7XeBjzE=>r;iHC z(4q2<8ZH9o(jzbLc43Oe#Xx0)9CYn8n9+mkei_i3fK!{{)r(n_t%Fc`c4Mf4yU z-iu-j2m%Up+!AkpWmW;N*@3FwQuzDHwIkn?^@7dwBf2^7WfSnTN!MK$s5-M%(j1Dm zNslEdX9yI)CP8kibn9q=P4^E$Catp7shAbSl+i$i;Me7>YLo-YSGP|Nbgo8to`opH zjzz5ktz(vs77j}=4L}UtNE==wvc!vR6Y|w@@-L$V!p(hF_imwHlv|q^MuqB-KLl8g z4m<@3G=b>mb+DXp>JYlq2;vZbdcRCkleiqtAl5*q)k0`MTs%i)XgdV5JcOMo;aN%3{bct~)Bc(3|tx)27 zcLOj~G~G#rJ{8SjUvfxZlQoGGsM4m3-8U-H!>&R|CYnqGQN4o_Hs%pR?3>n_HdA++ z_W)n|vvGaNm^kLzMcAoSwMAQN23bd2Q1OrQ_Co`jL!K>sh)Ai_L|PvN#UKLR*t{VhEqU3*e)mv>!nrQq( z?2)OcxJwNBv%74i?6{2)@teWMcV-1PySBmw?GuAq=$EhrI$8}ezyH9&0OqOI3jG+?hkDY8o=J|Bk@yPbBQb`o+5hGAf3CUJyW#LX9))ko(Q^$M*iVmV6W?TNR9aIX}7(ZmP1v6^}`?=%_M&Bwg z`Z#dgN+B(<9|6t^fF(RbZ71OBEBd$@UED1DSfZ%p(1IoJ2*vyyDpjbER4B+fVWFHg zSL9(-mRkslbV=H$$aDB7y4rmEdlofp#!9o)S3#=Bz%5pWIs&X8E!!%5k3D2suFU{t z!F#lCQpPr&MSEJ+-?U2m4XGxf2@>O^Jzv}Q$jI$GDSf-zgF1h^H?l>0%r-WzSL_IC zhp~8b;I?O^SH~mfSfMC!`9Q=HIMI>O-kp#&kdY&CIqc3?7K&uZk|VNM-=V@$sEh+u zZ}Op8D}vyRzoYP3JweT$7#~lZ^CaaQ`&&9Q0wDdTdBc zinOKocqtV$_t?%?u1qA99`DesOl7;n1Pj+yoO*no>G8?-Ao%x1;8XUoz21x8E!fNE z{X>mNVw3H*idLE5q=?J5$z?_bZtW!M?B$p~Izs7$Px`a2wZ^N#z{?J|?_J9x9qMvh z#JGL4bNT?)J1jB@DVUYwB+7+Don13T`nU|u@3_$%#le`Nv95ERntxb(2}T2n(;YVxgS%5pSW$y19cd-~ulfNN zPw2N##RC*K{m~>-VkwfL_bwY=`$2aAmx*uUf`$t@e3l7GPYpw|m;d`IIastSmF=5T z@}GGCy;@iUo%(;6gE<%+rI4|C4xacG$oThnS|a0hqB(AU3hYVRWAN=4v?0=u?LIlt z)*>|4=bb|ovR2*=k}3VZi4#DjW)?!FELNK{%15J8KYaIJf#hEpJzui^{~0Dd>4AZX zw)+JW)QoiJ7VVS`@V1+cz++pqrFpD;8JWrK*M?BdC;DjUx%alpiV0%Q{gX^I-;gA2Z*6Cn>g| zj>xC^>pk_53~epnf=mZ0PX|2B9>l(qTjK7aq68QE1dHCEcCBW@BV@!LAXL0H5RrAE zjyt`8CP9C592o&@4ct}HR>gfPk;Nkw5w$2|A`!i0eC z#E0@RG*;G!^Zy@m@k*nx_#TqQXN!!w*=OLgK&m;(jtsz=2aJC1XZtf*#dr^b6wqK(J*Q=}A3_&g-ks~vt6grJ8BC?w zX={kuG%J+mhc?aWsWiDZ&ChL`|0WGazmNgZ%{cu*v9~iAn|TU)X?1bJYE%}q4?UX& zskw^=u}Ek%pB`wx5akK_N&}2tq0GnaU?c8m2qf@|7u(|uzO<9XH1FpRgJwcZ?`(EM zC5JykJNPScT;C|=DFP76%?c1i`W>Ihp!e|*1A2jH^pdvGHbOF`vFrCnW}-^MF?Cj< z*Th+GyMAW_BS(p}T>XG*L?#-|)A~g4FTcL&Z4ni_l=q!oOYD4rlugwDOD~VKt-xM9 z`;E_pBt2g{mmqIHFknf}T;V`p`lzx5c^62kIxFw{O9MlxtNoZw6rox@Jx$DCOpMkb z1TV0`fyzZ($?UmA#c$`$)+e3l41z+@&&`xPjd>Mf&&9o!ol36NrnvXL%Sn(}PouwM z8g1qSh}Wm#3xiuf{@mc!uc4?d=0l5~z=J6|t{-0)brIRX%oBK6UDOB)NSxuv789H} z*8eE6NxglhqVA-7h0y?hr@WjH7~4vK5jIKBq- zyr4UbM3uSb8x%sJZvKR2sAe;~``(t!h+HDCgUHj_l>>tu{Kz9Q32+h+~418*;Q>uSG(kH7_hCPR`S%lfw zLq>rFg)F3^b_|FS+>XAZ*8VMZ^niE2ib>y5TeVQ&X?Dx|#Djc;0n-im(^F{|cbgVYVXFQhHYX%}EJcq@?Dy_X5Av)XZ7zmwJ?$(>twHrB9Zs zq`~5^>GN9CsU~kF&v+ZmP|SQ$>zA4si@!sThQCWlkXWBezaTCBjYms=ok{QG-Z6U( znoiooBf1$H`Y32*Pa|DJIxPWXKL3NRSjp`}@1d1|J|%TDy-1fbyub~BCWaKQ{h8V80Sk?O-(&$ISSr~2qY8i)`SP82?g}*kEdmO=!1EdcKX%YcqXxz!#J(UQ~GJxxqC(gYUvAVb2$eKe#kUm0gKkr#K zIs(r(fK_&BdzV@gSvCduOM1FYpHS?c)S|N;CGjoRJxe=t_^QuEJYWkz?+qbEtpBdI zOpjQfeu|F1OZYQ3yoPY^0NoGS(KGae7K|HCF(7e?oWU$YRS zh&+n$zOs35QQjwQ-m$5?SKGWd*t{<@q%Xv5-gOM6_nOVyUwNn6yoIT}-?w=?BT!Rs zMW(+kOTG6fZ^Y(ZFM0cXy+@a^e-Xby1PT2CbE)+}ALBKFB#E{}ix2MOBaW{aC6AZd zC*Y13`a!bXh#OKx{2+al0qpoL{S4910R3niH7=fP?@3Em%hIr_1{CB}3qSe#`8}VB zj!Z;BmOo&e(zebr$FVqN}!q&qm$QR(i$rl zCY6iz;gh^W04s4qVx+n~O37!XC2tM03Emq@eidc@Br(dbx;6is5og((q1%~&dn+bb zg|D@YxFkvAPNY*JV6Iv>PD$c_BG}Iad$V#?1(NtTm~X{E1-7cU!f9PwMS0>f?<66^ zH;V{!Wr82ms+{mr?~?}QY1#DfG2Yt*TYP~JI6AXdqEtT9c$(PMmefZoI|dE${%VMh zCFNenN%SygWfw1l58b~vL$who7|bQnvT|ci$88^;@~-(*_)NZ z?Rd;bnvpGbFNcGSaP*t>h;FgFm$!n}-e2DEcQ3ExuK5=Fm`lA2QU%Yo1wZ|dZxpXE-nWvy? zlN4$TJ(Zj$GHh1{cbFz+_i3G&!D}Rw_c%jD!IF(J@cL$@!X4?!^{1|Or(|z>joSNX zi^vg!!S3RGP&ZhFmym53^tN%L8yqbBYmRW)-gEg3y%UY)n8$N@dKA4LMltt{@=9{O z%naH~W)`@{8;*{?6mbyiV4I=Q+%ty9CY+HIEJBGpsW30xg`Aox?8VF9Nj{m&GrxA+ zh;a(yx|&*CH`6sVeqydOz9h#P|Cp#mFq&IfN&z{+NNuh;!iEJ9Abx3+rlR)uy#Fz} zb#Vgvy-1zb1`Y4`=$$|2&jUPgC-7N;=1?^s%}Yr^$}T0JHJd=S1ybht6XnH6Fz#Un z6azils!EOL5jJBu(ZZo!uiIW?XvdUL^nnIbaJ}~xI|$}LZ#^0WvdM3Y$)O&0U4HX<0`xhfX%5##lz)B-V;iRIC4<$)*P3Pmw0 zg$cnRocZykfoOkt4n-h?Nuu$A;NucARi$F3{-=h@eDN zm77Ht_<%VrVz(fZS{IIw=p+tNR{z2@`qS&SbsARm`FnV@)iO4N8))ET5E&qCJz3JQ zFMx{xS70qhD@=XC9&T9M6f2j0^N+%=Ww)d zF0-IwhZB27FU{~Gfe3_vhr0kT`gpEt@GWYV@`^=GFDbm-K9k>VLHKPJtI*JUlJFY@ zceB@!I!X9#kv^0l{DuLTMdjjGRe$z{trIhz9%XvZ0FZRT-?~x}y$49BKXpnh#hd*S z#K?WT&`@&M_;E^34s(X9jZSz~RiaBN2P;MB%YmWDztn%1kycy+LD4BXDAF$dV9}~+ zCx@aRhhl%W%wUI?|0nhN!a`26CXSz)cS$VwHQmM0-if+~$>;`G?HM2PP6f!r-3y<9 zhMQUFMoP?N)Oj-6(io^Zm9`9xO?rZ(icR4zoW8=QSS{G)ZGZVt(3%F>CqotWD$=P7cs*~L0kaKV*LT4EEl;+d$f@h7co=Q=OUlKsPS#V zMT|rR|J6*%^VWV!0?I^5VExS^%0Hp~rcBRqUBgCtfyw1s;p4D^)jFC5(QVcmdBLdX z{n{^Jgvw24NfT8HB~OG6TOJZFbjtBNk>nv0xO_G6B!xk@Kjl2GCUibUiW9uo7Xk|- zG24L8_#EU5PM{dB!wfAE)z63hlL31+%2l?9QrVVMtYIKBVmOEU*Tg;q8J|b|;wG^R zlZ2M<^Nvvx%RYt^m7Jg{_HoP|mVKBBkj_5-iy;ESN%o;p5p1S7Z70!nri3aL9aBzk z*&5Z}V3ze{uIING>W?z;{c&SY|)+YSE5-ucRg5A{Y^HQnKv8gVwslKOF_bb&Ur0S4L z)!wG+Y*P(Xs&|!2cFX@@=fiTiZO@pd?Rw8NEmNtUClws-a;Ba)U@83UJBiMY<#L8F zALG6MLXyh?U<25b2+;!dZ_M>;7vXOW(JLR$Ioc>zXX2szk!vzCp2=jRC{mk`Q#G?_ zE&MyV+>V#gnvmPeoMQH!YyM>`>k#bRIB_ZGnr}eTy!|vN@v1rRYzx2tWwl*}52bK+ zJ}4TWmL;r@BcJ6B!b?uHA-V>}>2UcUpq7@%W+qd#Tpzbs?~SI&GA7y>!%2~C0(Vu*O~KtYeH#K*Ujxjt=o`kol^d16 zHu^B#9wujl?0mJsSOyu9`EN-|5N;?ER_5N6>kjMe&dx83diXKeTNV^{RS5Vs#a3FdsA2U>MXW12XnI?p6JT-%3XQ9PS7jY;_*7x z9Bgt2BNtj~7=$g3)x3bN6a?Yb6Ae?qo&z4ELkjOZG(s$C5{-vgC0%yHZwS zfVQ={U6Pw!9%1#@vtZ^)-kqbq#vCBr$$QScROGM_9BoQPaoV8L$B!xO32V?YafT#n zCw!A;FRRlXuO6=iY+v>L*jJ0D=;-a*Ob$GW9#57(Ad+|2^Mu6utdOP=@!;-8Q2|cn z1>Narf02;)c|W^Ad%Lw9PnVQjfGPdOvJWFwckQ7eY;X2w19 z!txQJVpdy(1uCoCX$~T}_Bka1xdSnj7Mj>zSrJ#H-Pa%7YlfMkcKeslDgI{(DKtaP zGc`k%&&*JR+@1qVbX|>e)ruKh+t&@JU zo2Rw?;#kcJMaHyxcYE8K8ihBe@okj3(yxhSoYgx(Bum&wNnt4cnX4?i@d5d|Z!r3{ zuJ;hBa$>n>AsxPH$du~wNrVWS=k`b&jCDf%?XieBtu)0@E8)-#Bna=$ zeU=u;2nNV&&fi*qoG4CA+cq9e*3|K0`W{AEfd5oNje`{r1UUaz$;v>|Kh zQrJ;CHAJye?Gici|q1=z_r-bm#Pfo?7xG&~K#>mSf%^BP2s^I5~`3 zCW*Zo0Wk1gMe258Iw}0}6XI{S@!-7WC0?fC zeg1@9uO!1;6!Bm>9bBRK^HTBZi%`Yuaei7mo=~D=Qi;4Li__qAuOfGvJ$-`^DO`D! zjRr^oSYXfXv_OSBOk^FAkNYa~G;Eqoq=m5XC?^!f@eIUlq7QFT1-zO{vAu}43T55` z?kTWs`8zGTk`^_CZKsU3!~lg-Uw%w{zf}CuhT#wAq@`b>^vBxtIKN)&8g8d)e2^f1Yu=5k|%0K>u6 zkHk;wy*qaake8R3Wc$7I@Ny_79qaRypWsH5U^}h7+e%2dti!$I zuK~4UY->|FL15EpZ`Dk~C|O}GRIh#JsdcHfb6>@?Qnq=Uyx-3IstJ-irmYc3KZqKi z>cR2t5pc`h3+r`$f!VXmJK_qRvB;^DZfNe(?)+k{!0bcg5ON<;sN>G{l}e$axK%3c zms06sc$N1S1~9sMH8J#`rY_z)b%+=WSHzOxzGQ~-srf@=q2~}&8VuTpk7EZz6(zFG zH%;{}{*3Xg`2=m0_MLlmheKFv^BKTTnJ!k?kukMx;^=v#GAMsrD+> zM5U7A;;dAv?lx5)n`*OCg-L~A{Kv;o7DOL5#(R|jA)~cEU|+gQV83@Rc6Z-Y2~3n~ z_Ft<6Qs);+FSi;G^kq8TX)m(f_=v!~tX}c}BP&%r-{Q*(C|%026+{EQdyg9*He0+( zviKT@(yL3aS!M|M7Vk!awO+mKasPa&5i6E8pPy)jK8Zjw zNRl>3P`}U-R0@3dzeVcU64D|x(wKMiX;Pi z${+H=u8H+j7z_+tZrn&0+vOuY^p8BqTk6fDED(Y#vWZ8PB@KYkzk_mO9iSq}*hmtjMIkY$F& z4C7D8Y6jc;Y@QuoMqD}wv$fJ!v5?yWkxOeX6JPhW3=~VCSo5W}a?YM#phu0@_WJ6# z-hA^-l!tR^MTR>hMFC^&9CDPYZvQ*iert=?7VR!nMWG@&D{XuI z3RS{^gisAz7NQ>nCfDtIl?_xQBqHQde09~RBp;FnrE z@0WRH`31EP{<2fnZ-%>N`Q3*_%Qob+h;;N7agO8QV0!85Yas)JjIi@K|n?O}7nP(9nif)7~7~ z%RMU1Rp!|C9&gHv)G*oBRBV(<5b#40<}MUOL+(5?BJK+oZ1oNIJqO z0BXSYc4S90*dC=7X<%zPADZE3ivOqZON%`k{L-S22ESDF{{ntZ{|tURTEZ_?X4yC4 zw=o4j@J})Z$Y=Q--xiDp`;3J~3j6lX#@YgTCO7K1Sq)safW6aF%q=m`O5wZiOM#Wa zCTq8Sb=K`ti{m;w`vW}L)P2>dz4!I=^1t-CY^OU7%n8PBm&QC8yK|9w{8mCiF!sD8 z4}8?!v|K+cg$>HbO0xV?w_ zUXgvh>O9!?z8st@->c(_dYu z^M*-`Gcsk!tF1|#b-Q3-{frZoZ-NAmR1l8PNF~^$g1B(=RrQT7sxPIHrZ&c8A0MJ$ zmAX$nMOfr;n?NX`Q?lioLhk9Oy{x_H-i6jXR5>@|X)Bf`%PaVV!}Z@rJ)-+)BF)00 z-TjlCg57Y|79}&G`Ix=GAuo@8@1(zpt5?@0u6Tz6p0+v5q6eR9>y09t$?@Jv#JIn#S4Q}<-1w_+`-I%2DeZYI;G&0QGS(A@SMv#HZlYyo-bs`xUJ z8T&`BGOK@91m5h~_yR?%%f7DEup9a!`Ke0x!AJ9(sayHxQBubE~Z z9Z$SNCU9u{ypO5PE9r+nQMb>pUxdxYI^Oc%;&EdNOUy0T@E5jav_Cb9f?^#XrAMVp zQP@hAW31BtL=lo=@p$YeK!*3#@5NM&p@xj+-u#0;FD$fX z8?Tx#_J~6WVQ8{tWcXLYyzC1MCY!hdy!LgoHmV(&jcmF3K_Tx@K?CZN) zK9~y|A^fv>sHh(xO#72b- z#46}S=~7T2MfuN$(R|xDb_enY5PK7WJcVPQ|K>jdv7N^Ov3n>wZB||nHaAoLE6rDm z@`w2JPW*v^nD>-{*y?+a3bF4GH4qy{CtBjzLD2NuhS;^nIY^)1%LXbbnbrLIKLN3q zIs&n4DLM^e=bK5I#IYiO%KA8V5})3eJq^U3M>U8%_Q&5J6=HjW24aPDq9u;igQnj$ z#7ZEP;MhbNb|oRUaM6DPV$ob6)|;Z!AeL<=X%b=wd<66%=JDyhbDn`%qeASCAqHZb*pY5IE5{p({Ov;Q3sy)#EQ`ug5Ig(M z{{+NNKL&`cjK}PoN8F39(+_ z@V7lHSFv>f#MZLykb>C3JN^?8`#cAT{gI;6aBQ+cxg^AX<|CjFv5|awf9PQ#w(GA3 zVjJ#0D#W@EG7zhz6D{SjCeZZ%A7UZwjXAP}n&$S$q*IRd`7?$BDR8ZH{}bS=a7Li=8>_lSKj)%$e&-be2@*!Lp6Ut`}-(fg(Ly_4RD*!OI`_p|Q@ zS@3!1+4r4#Kh3^x)BB0M$1^l`c9%gI7N|ObfVlbj*idq_0b!$LJQX)TD_ci>peXb6 zWb3Hc6=i@2@0{vn-B3z#2&t3^Pd@@94+;7@k}GCDovF#atPgVRG23PI5f{QPn@ zE9Cp>A@vGD0oR=7IYPX9ekGV$cm}E-Cn&C;=5%Py!pSt+*BZgTR-lSw(@k;xr0cVn zR;zz)%_bBY&x{uP!H*DUcUU{Mcj||rEhMcwfJ;T%I8lt}1C?vU5iu;Cs~Bu8XN24- zjdhJhoX6FM>$#U?cc2pGGumag^vpre#JP*CLeia>6Y4iHFHouLp~|C| zVUTte2X;zw@;HN)XL8;~&QiBs%sCu6m>HNq4RNpBJu&EBlj9ychU=o? z)3SQ4O}tt>EH|+D^~CdLJE(boBhuIwBmJJ-2swa3j@+7so$YAe8 zL)+UznS1bTwawghb2pVF?Y%>&gk{B5vm?t}vOqJeF62r}Yz^lUHwn@uPBg19SX9@l z0slnjrSXDAj;@rk5L%^>(A)^60+AWL!tpe3EsE zJaMVUVbM{2GA@aZ>BA|lowGR+f;r7QoFLYw|4>5>-VM1u@E}^3$MFYq#?J&)O?&3iuJ}?t1}AdG2K;-<901n zRrSWN+Nanmcw3nhAr2jZ7&yf-1xS^?g^A;pnMyXv@}NWr>e#9{<1_(9v^JsR|3zJ3 zV5ItRq!A31Fo@}k`*Pu2ppZDj?nee=)7W{aTpP}17cBaEKROJwlm-Y!E8WB6JyEOud|%qV&S7FiVm{+Fu+6D&7y zhu3E6594pDd=&g{Fo!dktImUc$owN%v!kuaDIf01 zhq<{c4T+psR|aJ>yVw_|a4Fyf25r_swZMSQ@=sJ5r4H_I6W`15Hqj==i!)fMaR-Z!1S-qnLLs@<0wj3} zHbI%k?Ss+D1Z;BLD;dCLknhn*ccMjG$YTl&ROXVgT&@6aB2C@bp?~0^^KNKt%;_1!qh+UO42@1Y z9moTc!ou`HnX5TOk5hxPJ0eD<9j7QLz@6)u^yIVApB2jF__)Iy25j*yu%sk(b()ye z*cX|sV>s*1+?yCG{=8xsh$4cp#kBg;5j z{kfbI^i%rX-X)|@cp8_n)KKxq#6ut+eQtq!frWFK{qZP27_MyjBSNb=<^91-tr)s| z`j9b8d;M`t$>OMP*w?!hg!C?&M_Xe3e=-Fjao=3?eu>^cvF{h@{ayQh4tc#*_WcZn zzs|d?xCB2{CU3C6r&IA)g?*SKPS(#@{ak6%ArPoXlZ<0-1rJLPzAQ04{c~_)V*2O# zi7Rw(2etW=z?jLLcbl3#lOqdtCyFdskXLjw!UuIp#!I!XqDbTRnYw*I-Ijo&_PpX!Bv&HJtMK9fXkkFN`V$I3)&qB)@!;%x?39AgfK)$Cm0!HYeQ z!OLBR57-dv5}uHU3af31JsCp?4peN&%l7!xQpq+Vt|A)TyH;sTS@pqD#4?=>5dr3> zL_KCsLt6BP93!(Ks6}2c-x=ge`qW>%JrH?=0!z8~A~rcw_PQDLS}@i<1Y&)8a|B*& zMF9omSO&r|GMDLG^YdgK6#d$X4Wu$i?*vVvZS3!xTM)zA*xiDi-TyF+b&W6+i;$st zg8+DMQe{@;V_}CUZaYBHfxH~6Bln3@4eh7??OwI&H#tRkhUP5K%7KC(@cIf1Az<@r zH3F$ggcO?n@x4jR|Hmx`^VdIVF#mlXFPu-Cjj82psZ_7nR8QJeYnAE(QZYY%2yS_& z&Q?tvvpe7|Q)8Iik1-w7?5A4Dqxm$)aoah?gAqL~cF5Ai&0z}mFsD(WC$B1k3jRg~ zLHAtFz~?wco*Y}>94bW>3B?Ab^Fp!H_y0}i0!^Mbld5Agvusw}nhbD5|3i%2n@v(F zO?7Dnaq=9g?dIj7*sXb?Xg|%5Jnz3L!VImjT;`Ccip%(eltY)y*(1hD@zH^h$o=60 z<2B)!!1EwfDSq-nyRdfqnHij?5M=z+$+2TV1cB6mT&1;lUpgUs0oqyuex<=H@>zG^ z=QI2BI>|~s^d8smuu(+?9 zAwKO6#bB(+89%T@mITpE$GxF%uy|)6azCX*{7VCOEf1fn?BDgX51vJDNUxOS;g$xc zSj^w!dd|WScUk9D^mbfOouQN^0ZlDXQ=OufxcjI%1bTAB64N7$G}WBJAd#}Vw-z|= z)b5UZY0t8vot(BRje3edBNMewCY|iH6AREl)j`UNYn`^pab>K)&=^w*hbw2zn#1_A z+0_BXJk1ay_zuZ#81Y7)jt!LRyKw*a}PRA^~Rnu7-?*O^Us>$?YO-goy^``5EyRc>e28)01$5E*3s}mS`cpJ z!qM=-e7zOujqZjUyX%ckhZ}q9t*74l>P_CjkpvG_JPI($ByO8287WX%2$N+*0+lC= z+uA{W>Q)xg$MRho-BTLb%3^+FsEAv7R;1xW?Sr!?XRXk0Xa#0?vnP*QQKH9~6=Q23 zoV6~iq4vRwb)g2LZd^C2fvBIa8`D74>^Uaa9Gh#7%{7NyL2Ot;#cx;qrCj)=3%%pR z85pgC#Vw3w=ig}CA>NtijnBXkBoayl(2EU6X`#438-nW@7YAE zWO>=%FH{(WWh4aV+C}mo=C!}Y1YH>)MqEq&<8XVoI|tZJ$paB1GYT>*vg2nOMQlm; zrmZ%A{R}_MwdX%9OIe3l;tfa|Vt(Xw{-Tu{nRzzNw z6YE@pvEE`%V1doC!C&w)YH?Ow*I{2)CQQJ z*5;}^(-YjqUYKjwX9ScqIF!J`qpY)pNfVm znb^GeEoLl^0`ay^%;B*3N@vZdPHaY#vu3vwSwYKbPbiiZ92U!pgkrfPv3zxd(Y1-w zWlvll{gPuSH*(Owt6LVG;AZJay~~2(Q^rd>)hcO^S2oeYsth<%8gu@ zloxsoVZO&SK*8JeS5Q49mL}~#A3<^m7H_CHp%k4=Y4rM>a<^xBw3nptyz=P#!J5ws z%8NGy?koccwKccg)EfTfv70zDv-4tVJ+~>i=5VskQI84n)&y=pmW)oU$7A$s8eUlj zL_^O2*>cnaxzEs&^5XS#UW&&V6+F*(oOLE-0G|(IO0P0je9IPr)FY!=i=#6^WFWN| zZhJ6y_GZU5=sg%GVW8ZwFNA=@zD5P~g%(9cn;lM@)-<(oUQEssj`1BL>vw0~pD`y6uK8}gkEQy%j@))Lk#xcef#iUYfJXmXIAtM#fES$z`OX8*v$4voHb3N z+TCC+BUzXpp9#8*9PEZ7Ic3q%OVkMBM6+I!GNWd9fjmUq`2`TFJ9yLEq1XUaE;Ye5 z@!_$~LDg=jIFMdLl3}sViv_Yl?oTO;E|j~+qa|4!imrqcWlpcgZ2VEurTW`KerQdv zJZOzM1#%<=y>cwQa$3_XooR1PubxoONCKQjO#2t~Dj3WCt)}oWL$LHV>bGeW>wDl3 z6l+P6Vx5?{tkwJ{LBq5QN)h@XV0gW2=ZKJ%Lc&U;gPGuiSW&cthUJwSB1T+LR4kmt ze^sFT!$iQykA4V>1qCaO4g*g~eHLcg{?O#k|x?} zXd3=_Ob1f=)#bsUPb&h~BL3f@Z<83@Bz>bhqHj>MZxXmXL*PX7qIvP@nxicz9wNFU zaVUK@Cfc;{H%7rogL@o2MFDe8jx^>6Vl0f2rW6C!h`a{F(@J|!F#2Aw=BqZ%T#F-r zn}V^SO~Kf}BQURv%VTA2kCaDW53V^Bj5I*Hvakuwogbo-Mm|P{Vx8x6Pu6z>cPy4j zS`#Nn_*mvI;@E-2xhZ_+HUJUV!uioj^9{wA5BO=0VGzX|1CbsyNkAVWG#ipN?Gy9C zJUNql1cTA`sNqVZLvkPrhF+WU;Hp7))O>ixLQ2b;??kf}%12-Em$SGkV@wBpB2MbS zwbEHv)yv7%F{!IlA_DcI7cTq>}Sxj?%B zb!OmBd0J=2zts|7g0ZWRZZ;!N`()s0%>6WTV0&}J=xGywsqd0BMQ5}L~l&RW9# zRJvtsdNqoU_$F{fK-tqg9JS+K+FuQQY8!;51*0pVFXge(fHz@S8dagzqv*~-^sYG|7d^-tTu;9)?K5!QWeQr=5nPXMkQhSWh-!?Yv}o?7}&0g$#(6 zy+)C3E{SHnhA$Gaub1P8H0c;vx$}t}Y+D-m&9c&hWlr?-#A)bHLEi~w(cbgRqBBz{ z8CjO=6u%RQNTEt?1AS!Np?4Nr`0q!%*A`qv5W}Na5H>KV^1blWtuUsldiLew%5RrP zYe6y2$4K-6K_rue0GDA9PZ|uFcG@MetS2QqfEz&iVc`bdcsLkcN2@cZ*V0xqSK^n@ zXG)A~%R|TO_H+!snq;K2PETXc*Dm5*Y6-IezO~RyHzr8rQ0{EB?I{M$Is*nhyqQ7o zr-G>F4E95}2Z!*^$17mNjA%bM1sIxrB2n1F>w-svi#HaNf?bAL{B%tDxB*ZV`! zuFSuh^5WIAeh{+rudD+!qpO*J_k$i?ON1WxXH932HYHwcrq>wjm+r5DAyXfOCw?&i z>i|PO7%Xn6=%(HbkCkP>+H*?=bBc{g#LqA{m-=!S24TD$fz?Xp&bGitpl9*KUy5Ec ze^~pk46gY!DIJWu-=CnCKX<)s zYUa-)g3&KoD+K2qo(Iwd?&u)eDtA#EIhW$Z*gc3MIoH=UYS7@(rx|=O>xn~hMf5kq z&648qT;;!Wg;`SUhD9IY8yPMO@gKLbIYZsfyTHS&1^jf@0%5h|&Tix*cj+|FNS?cL z+DY)q#LrFne8Yc(?sr5k!9RWTHSNS;^dcwvlSXqJU^;xF#|4>`(w(arO23lQ{Pe>b zt$HF2p3(2%FP5!~y%NW-T@-XLQrnh7al07*&+Y{rdPIJ%V8GGr#+N{)Z9ctypr&+NIK4%ul_Mr)BD#j+tr9JvGl84^M`2(7;<^$wjYhs320|=HX4xAm+H2Pd zY+xPiG;>1hl|e{5&1)0iOP}BHhvVCnyInU7k9OU_s!mISb6{e}m%C#&z{p2!u;cl= z+8X-%>uGN*Lt1}(t=(UfQ*q|emIL^`zVP^0y1UPX2hE^|rxabZV!l$g-Xey*Eh-(lk@H0@6Vx$ zNA1r)2>vJivFFQ>oNK)P6*KJVNilqPc%C_ciMMT>SX>rz*#Ix9yCinubIijG?`cYu z9~<{H^)OVl9qgEWtkEBT;(c)gku`_gr1s**lNxn3!pD;iLIlsh^UO<466uovcp3Xn zmt2cZEXBX1FEa`;;GFH9N+ALpAMLJ;=c!9AhQGv&Kd0qJ*lbgjGTXa6O5N@=1(3&( z`%-sNNp?@Giu97)@LR^vgkmqrD2k+d5EZEENgBs}$>ZB`7w^%-UR(3d&zpx5P}vl* zh}j}Ax7FON0^Mh-$wH+KJWQqRfizb-j5IhoeM-gBkj|8&eM4_K(35&n_C~~Ozc(4b zNb!I1@leg6^~&?gy)xWg!KOTahz#7Xr<}MJk3I-Z;0gu4$&cSc?FhXyPaZgXdXaHB zqv`8~#?!G_pFIn;wZf87E;t>7HAyJ4x20n0_#yjUe)ck(y`#;0u$cfW&vSR67nKYHyi{&7J`;J)Qf zpd7y_p}kO~6%}z^m~?Ev5d|~bVZ+h=3QexI9K`U-LAd73&`UeNUK`s*7^b=M6{rKs zD8M4eOCwToTWp*H{J4@-+%g-d06(s0D(+z$rvN{$AQk7@I0g7|d8xR`HckP4Tt+JH zQX8iLKW>lwzL-9oZ{rl;$8Al;oowS2;K!{_#WhVZeOG`VSDlLchmBK!ANN!$uHMEe zz>j+%75A8pQ-B}0C>6KR#woy$o12Q8YU32($4yDaU2fwP;Kz+k#r3ms3h?8cRNNMj z91_zr6|l+%tb%YC+kx(q!3_3V2U5q&}hm{n;yAFoyH}bByqg8Oz4%&#&?Q zy%>CVyE)PG#lFkAS8YlQO7%FYMME_IT3jtQ^)%6|=Bvdn?ZY>=Qg&vP(4>xYz>i|%=M6o9SHkSQd{ zcumyU>_3{hSsQfEhX`mxl|NNk-@%K^BvPdxkF#v(oh%G}=de5CR8h3_e z8(PoMfEtYmk(#4)ON-4*^$V-r?*NUp0p}?Sny;f+8@9vtKZ2@gOPsE1rzTT`ch6@b7iXL&O5PN!nX}Q1vFoxnPo=((~;XX2OWvv&o z8M@dE=aE53CbrRBLA*U#?G0vEDIMl|U2L`wc~H;NyJU-iL8`53=?6_sCC{NA3M*ub zw~6e%FbM23*;LvkPY=TLm`cXxMLDtZ5Vx%8s5YJ`4EY|>UJu4bK-eV=TCQ9dTy1Ae z8={l=?mGyW=r6zV z4=na!%dEp0DD3=-_gjMZ7I4LYH}97k(=JcnV!mZ4stc7|f#m`<;v`95}=f%pG z;iau)%dOtAL?$*T1;VuZN6=5nI0peOg6 zefhLKJhfUIAAb()PYVfDYX6{g?wr0E;r19b4Y?x0Kr|V62i^9(zFI)urV%~a_ZzUl zmN;N}C(}277ueDI@|ULyNiz~taL8E$TCD!c6gIafmwyTC!0=tO{JNG7rcD- zR??%Bmj{ma;y**(-YC1acz0caVc3G1h3yR$HcapnFQCFOPMUXECOZ}H1l{N7N>CvE z1k$6wb6m%ZPC6E2pBd_0KhwFu^OFkORs3+6`)ohk&i&z%(z&x~;&(LMD?ko5&Uj60 zp~;y!V8P@4)WVd1oRo&eJB_v=&t7KT*%kPMFBxEiw}cW=U(0jXM4H}fKORw_MkNG8 zxiEF^=N0~hB!(~j&Xg~MwSLAA%1Jg=2Z&k$2yfWU)fFNR*xmIRbE(Kvmtj#jO)UFK z)(tO=rGUgQE!{cia@D-d`}i`$m!429u|B^hnXxFsou6)Cp1;vjG+m0l#d{QtHuS8- z)Y4^J*wV9Ygu}VE>(kV|b)>0#?E+KxNY(ul>K1(h7+xuL8)AZG3q%H?A)UKuzI&@i zCuvENySz^(4Z=f2Yy6w_y^;ZfFG`!s-<%NG5Bb1UHB?j`DqbC^+zN6Bt=Lu5*w%R= zsi0@48^uSg|d9^3Y22z7lv@u+UJwssrxwPKfVnz#Ny9(5(H^RFq0K*Z~71X*_>WW~| zm*MWlV<7JI-|&tD&gimrY28pA4rq(?h_U4OJf@I_a12J9(0tH;0-L#+J44YT9nwJa z)f6sKyUdr?bV+>=E^Tt89V%yxoxEVQ91S=3;nw3jhdqATPVilCk zPu{$MEdYn}6fg+iPIArTAD>Ys_8!T2pyC}X;;#@=$}Yxvuy{FWw#UhggJyfAO{m-kKtXk5j;{F` za?!rKC&Xnnt1HHeUf0zZ9EvHA?kF!hFtyAa$-qRUh-r$TV{B-14V}Y7X6AUBy+3?> zsEGR$iQVVz6(HQ+X!H+YAy=3a@1l-!E=AY?lzVgfif!ptNWfD3?q>3^QRcl!Hmq}P zYj9D9wLE^oL(yBo8V>8|DpEKD$BIbr#gv$K>kvSB=7D6Ys)tAwpT|$H>hfrPuxKNC zVMpad2f`;XYTQtya}4FCjRcx1XfExPX>cFH)m^T=Q_<>1u;@S-{Meyv-yxbPE{D+4 zXSa8|K>^Vi*$M**dKt`I9*RyC2BjO&(CFrNhH>vs&p1)>Y0G5s=QI;z{Ws47*@{OZ z)FkoS>Q9p5OiJPPbJB=x9TrYn;e--V!C(DP^n;c^poK;}X{rc9?6Ifq8Dw5&drK)q z^Js7Zb8ldQu1t&#Lr+^1?#n7JQJk)S5tCj(df^C`fPZJ-K8R{N7#m`AUt7H!h)iG~ z1XXH}{!@xEhp`OCreO{-7=Y>C@SkAEeQCb<(`g}DT)QPk;_h$=!ywB7nlU(#cUC@ssU$;QT8Cx&vo`-t$@mf+47Zzc^j!omrrgMvPV`{>T;wmqmUVwm7R*v2>gbF>)d(Wuukz#6>}uyUvEaq# zXZs$ss3SSIlkH?4tUEZl7Ikeu z?!oem-TV1CXHj9mr2W(3a~JhE(M0#|HtxYe8M~WI`1Zp^i^>8WcyAcD{qR|fLML_@ zm)W2C`8s>isPh%Gn)e=yW?oE8AKEi++^p~YI_s1yP|#!>H|zVq&OKe9wr%6K6W{fm zRJ>^>{wWLAQ7BP`eN5WDUly!;Z`|IU1?xT<_nV6{&TJU>n;&JIxn|touFW{}y>WlL zI^)cb#_g>uShsUruRj#6+d8h--DjZqy{dYwa1}dsxqu{Jkt9WH*-QIO8>kdv?vr8|vH3w(ERrm3_>bej3zkXe{($r+u z9b8vKnuc9^QJQKjx58&8{mDA+e>skO%ZOm+p2(am^k0Ma1v9^7M;-lEO~tlxYo5rs zRrd#hRu-qiQ_YN$5--C{s=9KRnb#XdIQP#tGx8I5YU6_I*Mj(+W{GU{PP146SLAr@ zLz))5z5C0Vm8dZboJc>Abql1|iQJmQ>pHP^rot3s--OjN$M(v_OoJ530peYO$_^xv zW89sIu4(%FVB4!5Xz*B*zWzyvBk6iU$RleEW#n0A{XqJvY3r2?=G4z?X!5{wQWo#-Ze z1`g=D(@I!j^jfRzC(~zbMZK=N+`%Iv|A)P|fsd*>9{v+ZV3EKE1q=!r>>>%GCJIW5 zU={-0)eWLj2^IkZB33L28$b<^ki@cGSE<@ot!<_C*Q&MkrB($MldzirT6t6CMI;2l zyDUfmfrPjGzjN;0ySoW$+o#Xdr~m(-581st=iE7ObLPy~H186r zggE8U4dqt=Hm7p=B|PW1{V}6O8pn$S*+)(6ymSzj8YdNyBt|^j^oVBDZlq$e=lxhQ z*&C1ome<1+jM&T%&=7s<4YbOa{K?&sS2m4MuhiYAUb$3W`O~x|GRdH5iPLcL@?fpIFKlUZWI93ES z9b^x68OFOYo$7q${F65By+_VV&1cvr!~*W>939SnI&P2plG-EG7xc(2zs8Wf3P%j z_76DtqAHOPrR*H(jW!1RAajReRM)wrl|7nT{$_wh$fw~@= zzo-s!;XK_P!eLN5efa=x6xr0%6G!w!PHL;;bP}!}iJ4J2p_kHZ?iMAZ7%SB2IuX+R zcj?M5CudGu=DeRg&f_P!`-Q~dn%a^o=l%*!kxv5TIsN&F$-m1Lbk-B z6h%z-H#=}8FkOz&$uR;sc0OX1^qmkmZrCo$P}czi;N%TmkoQ@7KCZ8jY=)@nMR;Qh zO^!+p=gV$OEdTcAHz>DkrLZ?re52;gZ9ric7lMQu+Gy(WXx9%wRu*Bt03TR%ZXn2F7*8Wwho#<3G z;WbRq?N>8F=j91i*Xjp{%Pjv=9|uZ(qmVClf^CXz?$oamc1xxeyRj2kJ9?&mF-9h_ zTy^MMluN%9-6_UwR+Zy=M{uFKVQHSjbU9W;oQYUVbAO7C=u6lN;w;TGB9`Xb5yC)9 zq~jv~3HVXO_&fZ83{iiFr}$;sqaSZ&kA6t9ZlyNdHi7@hc=$tT0v{0zb+l>9KQSs_ zk8%6&w$+YO!xss5;`ZK+-Rtgj*|C(`iT_pkMyV#23E?Xz4NdzPQpLFnpTsMH>)Xp; zR@o1DI^njH&iR9M=iv6md3fq@jAjpbJOJCX-g`gs-H@KWF=w^?*VPcyC(NFWMWLM4 z3%a@o-=7x@YGeV<;iI&BA%!%&80(v>9WR$xp7N3`pFh*lX zLB9rY0S?Wz`XjqysYUrqklx)&pV1=SSQ{W7Heh6X09glCw^RRuDzG<#?s;5)8!|;T zh4~iMDOMU|Lx2!Dpm#kQI-3=>o{Q0D;`o79mV+4^d6eX;qC+x;+7*PRfw5qfeiQBf z=J8j9i*c91wYFNDlt>UG2*9Sx%4uCRl!Z9!X9kqD)4LWgEO*Wc`{I#oBzf8OE&grEo=dfN8#8 zI3&wZs=cf&xxgBd$P@Y4NmDEIc?jJ=RHwYZ-*H+a0fs}aTP+) z{>iAh(Lt7zsCl)|8DEvgSDiYZT8plPJ5^25XKswR=Vc#L^H=kxG>G0y-M`M~kh6CO zp;kQItS%HO`YEQC430Z-&uD+~Y1lkmnVuvU9Q1QKT5`=cbGZ@^iZ)|sA`5ssG$=3U zdoch#AjV!!F)AjGwJc)j6@i@1R&IWwiH#i1b@{LR(CYNq^)FDxN!zdGGwSix*W<;&}Df+_WltF<3%DmB=lH!@UVZk)MVYF-gjF8*}U^;RC&JW_>Ln6uonJ3pmKI@peFkU39Z;bq**j2*~ z{*?E~j$L9GCro40gJ)*u{UDW3_Ybo89Q%VYd`|emtip{8GX^GgS6}Y#iRyc7_crQ# zLU$bUt8;~4u+`ZZ=kM)v2RX`_^03xi$vt{S4Nm_k!gHOmFK%W97W+zBV`uwM)>7r= z>lV;Qzpt{Fy(`^B+91t0Uo1QNTb~%WYDQoYqSA+cN-B~hB$huw2L5Tn1E$|A=cN)N<2GVxP~}Uq2%SsB*%4(#-Q$u zM%)LXRMJh>#G*m=H#?%^UB8r74)s`Umf~T!Ezc8WZ&xYUZ`$Dao55cetK2*AY^X!cYolbB?7qRJMe%=%ndMGo zw{&8bnD!?_!MRzo=#I>lu2n43OBY{O#Do)#S@c+CH@t-E2*J(0MtBGdfCLm`6SC|i z9SUcgjbg_DNT=u0>FX#+@_<2wBNMX#=FiLI=w=b_gKkHLpH^XC}V z0n*jwvriD3@-#(;rt{>6g$k!3)XR;4d7E1LIRM$GAP~Mcc8%v z{R4Zl`(Si2=Up?nst9AFm&582cR`ugFT(T;|BqW`jqkSW7JM6l?*blD+iRtPEtF#| zE?W+BDE5qaj{%}xq6e~+5$Ht+(w0hs`CZj-_GK-xm54$m{Z4~#nH{#p$2~$9|X@*f;C$qRt6c-*9o+2~jo-pLPij`cV zxAq9%E-=h_tHS79n1_WfA%)?34+buLUARm5a!I^eKCX}tgk>`ulS1vjChG^Nzc^GT zhnYM`JID2tsxfd)og0)WaHqbUx>@vlD>@;CQC3U(U;d*>|nLM{1?nW(c4>Y0xAa?=i5qZvk$Mt0+wKT$RxUQj-t7J zGp}zG|`#bA+tQ%B0pk|S~>(jUIv^AvrG0GM+M6DHj)u&=LXvRuy(sXPbD?B$y% zQX&S1d&SBFXc($}Dum=4(t4|X_I%EGp*Ba_*G@IByDT5k^)T8#_I6eKuEo{HY~Ltp z->69YMn&2;Dz1H_*u@`ZwQp3UeFh1t_GQo(ao3a~h_tV3q7#XlsA^HKVTSwc6pdLG{*K6lo9xQ}sF@)U?OtSoL% zgWsgDReDdglI+wy>bq7ywJzQsqwII4domRglZ)Fw6N+I7aO zBZI>R)~ekjZq=SMLEj8IBpoqHqi3E%Abu!=zbs_YKT-l#*SWY-Ps?&;el0#RtPk~i z8)jU7B9@MaSc^?U9iB^1N6)xHn5*;^R~TFxj}x`B-9TBwn6Bb|6(8yd$D2+5bA@V_ z7%e0Grzena<3GyvTm0qc+E%o`$6l-d>2lQ+mydwP9ncDp)H<1x8mEvx4xfX>jD&W? z>HXL(v?3u&Zt&slCT`>Jt2x7z{4wzSEHSJ<A#1d1Z6#WaIRyOJRpb4KR;=iCL8fqd-VTp6?I9L^ zg4Te$qpNH|D!RrqF?cR)foE+L&xh~JDC@~W*0nsz1RsMSHiiaWgIf>%`jMO}okdN% z08R@thYTyYnpODbKvQEN-Oh1*E26rYOj~@IIMrUI_ooE*(8LHX964^DDTC$q4`U}t zjK0mu`Hz)f1(u~+sCHzA=2d&K)buvkdzJnwjd&r^1NW$dVAQ5kwJzRja!OuwwhpT! z<69?#faE=SV_}mionJGz)WJrUzLheWWviuo_{x~|4tYR1 zohr{e>pbYgpbCGnD&{CEbw94p1Wd~(vl}v57L!7(NEq;MEikP}kI|CIGSB7=Y5uYj ziZyy6+o)&^IfX6)6u_=g!FUEeP7#PWwAi20jP&^sjkdl6FrnSEnl~>1@tFiBT z*<&CWjXU$SmUDE@6hgvbwAToSiFpVk(40FhZ`(!0z|*p;+&Llp;9*Z-i1R-F+lDQ$!LkM7l7d6CKj{<51Acp5ZXVjm!wj0!i_3fv zh2(&Q-W%y1do+QFGe?s~dy9{9<@SuDY~Ly`k*tLXb~3IaRlUsg9qpfbl)afi8e=S` zqEus`y9g!VuMV-$PkG3G1hT;%5TqFxZlfcDDl=IE31!HNp3XST(<0*#`ju?vFi2u2 zN>zrb%&%oHjfU?**laU{WEj3AZn8ayI{Geo?f?_ZD}&g8r}rw%ZVdk;eF=^*gkN=WsC$k*oyHuDwF{k{zM zlv%ib6(A`$>*Hi-ru>~`+L(f~vS&5hw8uOI^XNL4My=;pv!_Z+JdT!Hv>d|<>3#uw zFVbo;uuzU?eUjI+I>|S7eG;3%?G;J%iRR06 zW$uqo6*W!Hp^e7mXwzb1^kibll;YpTTFwVdjIH8<9JylPMaYQx@M{?jy@;Y&C!}|W zdz0RR^2PoHIm5K-^|zpGCdpeYwn@?l^x{GN64gGbq&LUyy&n(qq~D7BMMy#=SoRAw zEO^gIf6In}F9Ci8?Wfrvb;@oQt>jJ5M~J((GnhX@9Qs0-QM%k-l9xcJx~tJPk~Ht_ zHdk{%n?hup5AFVT>ebF8?Qag~?JTR|c874$Cqx~SGGHg%=MiBi#)Tp@(d`Z6_d^cG z`efw30H`-ji`))yu^dRq*)XrI#<7pYE!HA)F3V*~DtGEUTg(OEgSb7%*r@k@QU*@tQ|eTLj5ei) zgW-w=V#&q(V9F;Fcax(g8&chErxNoR19XOHc=U&Z<9~+2-Akq*Li=7O;I2!hx_Ppo zNpM_@yRwFVqsrap66s$t`iMV}%pSwIiyZ55r0KW9&qT|(U7M0icb}*EZWyHbZpk&m zWVF>*#(0$-)^cj*W)qXnLH&~89VG-8%BxCOe!Nhs6hJAMEmrY>iFT)+B&>$1%uy&Y z4<}1y;{A}6H8Cbjy&9sO`da2u4FB*Uc+zk97GzdS_53bfD_!2zY|bngMwH$J z;yQLEt6Sp1^)!O5uD3}zs|j1sTCdnP?=?O*75(MGmx;T&ezd>mmJz;lM+`o1(bM5) zvez7|PAuZw^=!h-eRVxMBv%n;?u2Y=^_I|$9F>!hkpo;Dyl!CC$y-koQ@hgEE?w8- zR=n`8VAH~C(FNpRl>buseBaHyK;^}X*F|1fjk+qT+uSSYLl|9yYlBXF*jz#nuU8=e z*GqlZ&a2{cLs7>EJ2-Q?EP4ak0GF{8SFmYw(rU)uy_&I?u#n(R;h~XlLgX-7WmUQv zPbDhPay~IlLee2OC^y=OK;|BNcd93owG$m48j?DWP+Rh%TwjFGk9ZlRA&T2V%CpKe zZzv%ta@tXGeoakR0@m;Bt8{lKLb2 z+(!XPt7OMN<`72Jg9}IL5?1u42F#JC=ml2fQIIk{kO|^OQ*NokD}z?`C@x^At0Z7w zuAs|xi~o@~ZOPFgSH&| zqBq49E!bpqac%G7Ud8AtMzbnKRwWwWl_}i9!S`1k;jY*AJiGoPgRQjmh_P$s1*&7! zwKg5FhhyyO$qFRK5c1+b-wta>qbTneg~X^JNym?38i}SLx066bAtO}6Xf_$Y@Jthn zRXCOTYTCQ7m!BwNi*IXDkNm;YQj0h~HCKj1R`2D##M0%NN|XaxjK(W%l38q2sOUXq zoDu|{_rur4;=5AtwMSu0P4HZ5zKv-MT{Qaky6D>**xM45NLifYoamkNSYl0omgxLI z-)q%hkM{V#pufC~?tiwwMj^Mg>M!9Z%5ZcVdWh)C_~%>Bc!YXtm~&r@*dn37MZC)V zP_c=Fn2np1wv0Evh?w8yjX%VacW%2-CUq-M{EN}P{stluYT3aY!|1_3-VmF2_;%%o zU;LhUP^dgg3nf=Saw$%Nei`e3f6GNNN^_8jgVHQZZ7TbluL??}$Uj5x_^P*>mDk_M ztX!5B=P&f#_R9uZh5fRBoc(gU^lrTU@=|P>5&LC~y|Pcdy>eXCuo=x%wB7YBR?I0l zZDQr?(E3>AcUv*9K3}HFPW@MOr`{cs6|37tOmzXyRo}JxnKxsNyTU*JC41&L%0N4? z?3s5<*)6Pw|C&8>%1G&f|I_x&yP=fU_RQD1N$1t-Tlu!^neQ4LTn|7i(z88PPuJ?t zvwto98N<&w$)Ncr(Vi`YAkdaa)@ock?d7+zR*O@rn913{`u2bXgu96eNPrED#rrGc z7=vuL=4aNJrV!kD4=Ce?wUiT;Sbvex$;V~IJi@??oEsx#5Hl(t$-^CB^)xm#2?1X_ zLfNHpkIl!Q=o-G=UXJjif3d{=Gw1&$ROY{Q{-6EMqD=%279pZmujwuXhDWK_aPb#G z0{t>zy7$L;hKk119<^HkeOFnNW8*EFTWi)$owDF&9=EpE*uTUfMoz)OOpV**Bv_v3 zTKqGv*QZJi*o^R})X8R_aL%d_8g?_PC~m(Sn>EZ}T!u5}eC0-dJp3aGw;-mU>|>#F z;UV7jE)M3nGpla!52ys^!*Y?#(m-kP?MaWyr~6SNp%&jh_R)0tn($~=U|De&xj&{D zK{H*x)crBVUAR9+E{VytbV@tnAA7~#uaeHO(Bmq3jYPqkn24~haVkVxq zTN9hT|Dm+VCL<1h!!v|mL&>r!JxLzT>@wR zC}>z-LmZRC+Rkx-HB>FwG_JCxKYQ?GL89T!eyHdO(q1DO7>D`x#fzmeRr}g9FlOH~ zc(sB01wB^N^AUZQOL|hfu9MFR>*(YKsZ-bG%GZo_lY<8)ZV2v~xHY(A;;!H)6GMQz zsH|O#tX+((UC0_4kJn^8;^N;1Po{PV9+|kB@!$+KdM^%cVQqPnGQ)@y)@pm%AE~MA z$>E{y_Rm4Ry^+6LGfYj?owIMw1WqHQYngwC9mzO-s>XxJrc)$idW+DeJ%k&Jw|m4x zXbIV|W2Z4=aIF}g4hN9$3>h+dpA&O}lpt7Wti#i_{oP9e04QY@xu@#gSc5~zBpOFN856g?Z8G8Wug ziLhp`*iE^ic1m?twwrX$v1xBdI43bO?Byq*2se91a%Ky|jq*w1F4k4t#fs%%7qc?v z8cl+}?ZR|};aM((WeiX)JZY!_d!&W>W9PEJb21g6Y5Bznu_HmU?O^|xlYSE%>E?B; zpUb(z_n~GZKl`T`MYgN!QT2={8nNeS-hS9|Y*>Z&P}4%D3yqF6u{=X4`h8>go9*F# zE%a6C8W=F>1&8(4VX8|~LHqoiJw^D0U6r5tb!^9AAKyh5InN+eB-WGD5u+AlpCNNC zr%6m%AIP0#_F+4zz0n``@(SKWwA;Z~g8mT0YV}EA*+X48;7H%8ZcqpN>h-r_twxs7 z{uvy9q_Uz&VDN$;L|>2T3&J(+i1tOa`H}Wjq|+hc)a(P%0vA0NUZ%0rSr&WEmC#=% zPU3p7v&m0Hev$Bhx!k*)rVSpJj#W-P-;Zrge(mzagzeWk^zr1~&W^{{?mAz)`n(d*jUv016vF-C+J23CspI~=w*Y(=fV(RrfiG+e!wpMMjc2=Ta zxFdn*We-92t|fesVTO0xhc^nFqI)`xZ%#;9N*}wqlT9d(MdP?G_s13<`qIWBaC{m$ zIDIm8tRp>8A6hOf!T^w!=dq5g9at~7bPDxPOxQ8Ok)dA5a5b+hdMS8vB8SkgNIph` zggGdN0fXQS=W5w_8l)H>&)&@~{GB9VLB+9!=c5sWAgN*QS7_x8HtqOkEpc;rmzF78 z^vtY4u5q%6LX($LO7PIQHTq2YE&Q{naYgh^-%nIXtsxR&%f7gQPgv~u^Rp}EE4dSO zmxoR%=0mRwQC9@grwtG}A2V}Ct&LAlZ3dskwOZkvPcqt8x8=v!+UbRJ{+`kH?`;d` z1T)%h=IcO4+nsHLKiIx2n7=&~9I^dS@V@O$g>&A^=u_8NIOlH}eRRJ5oYAL&um8#D z)7TiO8Zzf&jx7$H^9i4q&)LjppE;i<28SFhubrMk!v-FFq;`osTz>G$+85-Z&%tMF zSMyMKcymVI4+;-|l+kxn;o+JLA`Kk=Dx>f2;Cl!67j8J6Ia3e*?cmou^_zJp_~(PC zc)D!n30@Br-h8_M%vHfNi~85{;i;9g9mTb|K{;!KJ6C0@Hlz+huvf@gows+sYI|CO z3#?BJ>~88_lc<&MC9y>F{nEysfaZHFl`p4vy|c8A?2B9-#9ZaR`X|$gW#-*nwm0$k z=cRiEi8~uJ-&+atQfa-Q@P1UfS6*WZuSs|?iEgU_#9&l13B*!r&J z=SjjtbeMT6cw$lUIz6!4W#3~9GJVxcy1ev>@lu_7=_BK%uzG1fFU?rTqE?kee2%?Z zY13>W+lslcAFZ}Zwf}pMNh3Ai`QFWu&ilydyvfjn>bl<5Mz={uF&b%jRXnDqfiDTI z2-Xqm$Q1zDRo>No{#@|=L>wR6b6~++Rl4gDf-=Ba>?F!utKgQ*}6?hb~FyWuz`E+E_&(@{B2;r-#{P&H0$siMPRUSjJ4 zH>Ut?b^FrQb`0BiAf8SJPSK)5aMYnFA|Fvnz&$c8dSh zl#Yh~)UU~tkI$b-AB6@+?`pGxMXw1qssuLAo8jRRdlfYXQgpT8X3c|FCwK}XNP4wI z@?i{;zZ?EdVn&C(vN?4^?I7uW=rk0b*AvCI>bL8!DBw*G_Z;Nm}OfoYG?QG5@ZUT zJLw-}A2|o!8}5kKlKjKg8_EMO|yY+di_M zcK;XpclBtebP)0HTKPFX_)W&}>4~a;cIxBRcdfqbfBsefu2IHZglde7i^oyozsDZ( zpYiV+2N|`thb&7HI;z#z@NL;cUKSd~()5Pv)SY??Y(O$S`FtyT$XV>We`%}U{?FRO zUu3lYv-aJ)e}8uSE|7txk{Beu21ULH$ z_{)0kjNoYLEBle!P!RuJ&w4q1mCi|y8)5Hqgo7Z?DCJPjg|XoR-_KOsLHNEH-|^G+*2v=6OZkd-fbq9?E@>xxpW{8s zr%01op;oCF#ve7+;II(mv#nZiO`^=Bo%(NhQx|?GQ?j?Y7(Af9YxP8$8MB^=zJ3<^ zx-jAY4*EKerih^OpP;YsCI1Y4ZQb}C^tI%--$h^T)TI6w>1z!$F($suchi^75dU-Z zwfRhwqOYOQnZDd$+Ip2*4|eK3C|{=0vfs3%FZR1z`TM_fx{2E2ff~oZaPg1wFMe43 z3qBRYzy6K>{(q$t{&W8R-DC(1`SE|RzkknTEr>I2|C&(?KO4Ix6w4n<_rHyDr1_0g zgR({x$a(R}h}LWasH-XR9Mgf!M{T^GefM%l2}#_|DlR(7=UymR9-v`NheK;HXtzVd z#SST)+1taOIDZYd*EsGdvEOvzn40E&B7NgPwvF^#bt z!tWH>FKI*9a4>ibngBK(&1>UqhB|;%L(pWIhxq07koqPhRB`|cV^W&^VV_h|pyB|P zA#0u#|M`g~1T!?1_>wOW%)mGxg8ebvHOBvi{hBuH_H7*xuIk^)Y#bLyWrH4)9sUM?HH zM!g{ywY5aRfixOlZjNHE?)JTr^c?SCNwk+g&4Y@#?y<8zjvOAoD6&3_4NVgkVgz76 z2gF3Tx365U23u+&3$LCm7y*TY7TQMbeK2vge$!9OOneks&hl@q_qD`=q({ z@`pGdt>1R4sX25%PyYZpu7J2S?>|G1gXk3zT4rH`ExJHoLvzX+lQB6~R0itj9It#Q zi3-|BY;pUaH=-b*W>2SIkQL^Mt|yP(}9b@*H(4opXka+7na@9kDbZlQ>4; zX<>hn{+v!$Bc70l^9e+jir$S?r{72-@qztA&X-78`IM&@NbhW-j>Hv5B9e~PyIxHv zB7vkdOK&JI!zjJv>&endzSi`j>0wCPK(T=x2tlz%&2QtVEJK1%SS;}s{q-z(irzL* zJZhqN7!>-?1as`Go?vK>7<#07t*i20NaE_5bvH?Ww5psjLG<3oBQ@=77IQJhwCGRV zFW4A>&iWDdl_{ws{5L=EEIW}lugmBuDL->1u5mW+a@HheO$fi`ENi;Nb3LDZ zi`omQZ}@qY0y?g5;MFDWtVBu+`vrVX_+k0#6e^~v^3)+T+kCo2xmgRZ3r#=PWXZ!Z z`#Yuk3zB7oHfdg5l1>XZn5>w&tO^A**N7sb`I-F0WYAv|xd7hq6e*f{_Iw)bS$@y&o2uT;V@dLt zWtn&yNy6G$ki2RSt76llaz`p{XZhQ79?ObYdcpoGnVjkfj~p3^E|X*Exa%l`D)e`R zyDEP+Uw?FD$uNK4Nd|ud3Q-d3CoM?gnC`isV!5Tc;z`a~@pFCm|nQejpB;YSh@aG*Pxqm6(znkEV7Wgp% zziEOUSY!>{J^}yM1pmzfKP=!SCb+!?o-N?}O>l#SySvo;b`xA`<(?tAhnwIZS>QVb z+}{MJc8b*G76ErN!N0X~4;Ap4!$#v}zrm>Y)dK#~1ZP@EE)?*36MU`(?keErCV0FB zJ_0rBzcazZEO1!BJ`?<~1>PfIj|pCDf$Ic(hY9Z7Ia0yR0v>6CZ5Fszz*m~!t1a+b z0!}l*Q!MZw1e|DspSHkH3wYllqfgdZ;Kv2L$pqW%k(xXr;CD^%Pz&r8@c)=#j|G;S zG4)4H@Jkl>0Rhi5!MiQ+ECEk8!99>rjM14cV3!HL$pYUk;HymVUJE=yz!#d}(H2+} zaFPkW)&dU|@PP)S@pghe7)4$o;LRp@rUgzH@cSnCG7Fp{;1^7AR||alO29ue!SgI| zNWc%8;K>$vn}Da7;JFrft$^Jo_(co+zJLds;8F|xCjnn%f`4Rze<$GfCirI-_?H51 zIA}EfSquE6fVY}pj|E;V;A#_mhXtN1;6It*=@$5Y0Y7Pi3oY<80WUDY6D;sJ0Z%i* zw^?9Kz&DxTJ1y`~0S_|4H(22Q0`6^sueQKJ0iR=n7h2%u0zUGU(fGS9uo|?_Oz;v5 zd{lA=P4N8|c!-SrUrg}tE$~$W{-p_i!~!oAaES?i#sbe4@I5BD%mTkF;9E@atro?; zBH+O$_%{~#IRQINu+2iURKT4~u-gJJ7VxnHM&qLuEEe!i6MUtWTTQ^#COFLk7fSBe zOmKk(E)eiDCioT$oG;)q6CACHnwK+7aJ0YGy!?R)?#vFKF)s%RlA$Iz+Eoq#_cg&Y ztRgQEaH>_k{w>Wb?*APJL{+EbQnm=Xl4L;8vB%gO3A_*DF z;jdWPL~m2FkuO(2R-Uw)#AJI#23QcRla?YHtwgG1CrzCwmpcWGDlZzB!Br zVP_Fs8j*E!C@>=HX#SU#4li1j6oa+YUVal9MELn3bylWW6i7ltsjarm2x3wG(lEKc z$xs`ez}jPQaI8Zf<%Hsyc1Z|c!GT2L>e?$d@GRX!zp&RXKfF7*EDWXJR+J=d z_73Ottl2ln>2K4$awyNHZ>LVa;Yo5Tl8N|O1GiG?e}s|nLTEQX#1pBkzJzm2le3$H zr)MUwU^9g8P8<22wzTljf((y)=xOymcIX-PJz;3G`gRO0JhUitkxE|Fk8hF}UB)-b zi~2808LE{XEYUbQvRs8*Ur%u_%(9oumOkzh$xEdqt~Ys#Z#q#c`?7%pS#sS_)Fp^= zY@U=Z;Onp`rHc{9^iy@;v%S0oeBtr0CZeca7doFGk>R`!w$ySsF?XrqyqDgWo)Sf? z#fUyfw}jRmrcV;tvB!a_Dx9KX(|*JuM9`NEom6zNrrqV?eKN?lnv#pakr?_lZ$wJ2 zhGUV`#6K#Eaj=>@>ZmpQIWJqt7RlSyG(Xe2(USR?WDga}6gMm;Z9^*nkgTF(n8jvm65FbEelu<)iBu+sF5FBKZD4U}0bRwE%NJ@FE$ zgq&P*VhA&6jnHNG@@GM;&tam69*-&FEf(}|EFwq^DX(P_zv*oiK`48xh@xeoK{1%` z_%_Vu6!ZKT%=U`mEG(gI4biS?Mu`8$dUrZI_Pyy=y*aRF)!SaN2XCiPaSZll z--i8_J!Y?1eSW#BHpIpWwcM;Y)ISx}y<TIGO5Qq9oL0xk&+9zRj zsc-C)rY~C7dR{E*L2;-*#t0g^7t|(cVl2l+_x6~w7R8h${UB>DeApmOKV|WQV?O7_ zmQfsPWG#(M!XYsjqAR&j_Gqc9DKvm%-NiC&Mi-PUk&3)#pox)O8c=AXO*e+ts7qoz zvc<8;N(9-|R>*$;jmX~E9apn~vB+d6Idpz2WVd}Ivb*Du8EftEiy|aix>WGh((!ay zOiSB+Bfj(F@aaz}RA#q5BFG+Wg={&(*x9>nZGBuzZ;VCuq#(Pz6|x!Mh-^VTvNOG6 z2J|UGRtK+)Z|23{h%74(nbB=23ZpQ+(q{#q2&3`%)?w~BdrRwf#kKU#*vdUG$Oxm| z0@>nkMD|2HvaYeno)csug;Fz1&G5q%8kt0)wDWV0b{Su*f^?}(G=^acSq>NzLw4H@r|YRi!04&Zo6o6)54--SiQUf zBV~(T7GtkI0Ok-XYgO#;$bR-l{&i<;BaLDoyTIz@o}unySdS@rT>dbv2l-$86 z*{Wv0)|tg(t@|URR8x|{?}X6$kj0vN0Z0rjXDx^=w$9e7!Igq(xD@+%^zC%@_CvgF za{4|B@S`Z8w*rih0wh{*Vn~YuzNY}!Mgct);L<1{Qvtd~0Y)p50K``MGNaO(uh)~O zkTrcBSKdZrRiGkvqW<0smG)LyAW3k=By_xpgzw9IS*}~{=%f|J0dgOM5StT^+Ncf1#qHn3gn$HjpvC(Z|@GiZ)uD5Ri^@^>bnhHgs8# za6VADBx)kL^4Qu|d)JdbmpaN%R1^-!5!ZvPvaOY2*)9Y{uKqXGjnVyOd&RF<8zSwK zJm>2_6Alrx2W_vI*CJ1v{_AZm^V~`vxi0Vgvc}6j9m*OnvsdJ**QhEY(1+d4ZKed| z3%RKLd}#zdB(+*Z%W_LjUv7Vrz!e#fI3!Q7VVtoi6+HR)n!-bKJEThJ3Y!FUZqfQX zcS!4vwLyQL1`?RqWXSf4>D(R>xqs3?~*SP3CFk*rMs-X72lfvF<^hXV8i5IcWv zW^cO%|6NZ;{SCT4)RsbGGdo&m-XfWQBAGXHB~EPS<05dyuAc=i5EVj=m6Z0m(PA2YIosDSDz-Ld2yQ9~0veM{aEj?L+VU3`_iBkYwQAFCh2 zeEA_l-!ZghVoY*YL^~O(y@ZK3?G_4@gv{$XB?LJR6ttaON{u|npA?@*%&6kFhu9_8T&u>+GD)9)I zXS64c_T()|xmF^3n*IRmWz+v54rUU{Dy6-%q-jFY4wrw7@&))f&%QG0{1JW)Ux22x zd8wnPBzawltDVhtHA&Y_2*2wrYrMoWfX}`~?SS+Rza(&Q<#D#){JbT3*D6PVCxLbd zKg#Dh;U#=_3NPmKlF$QUu_flDT=O?=v$Wt{wSBDl%`JSzBClf}2$N7d{Ogo$hFImo zG$)(>KV)Ezlm^6EAja>6Wo`A>YQ9&oFX|Zw$2=crFEy^DCL}m?WjPg19r^ zDqNmER<&bD7U#Y7`HKFATqB#SA4<#pL;A^crFwfvisTt<S; zH?WDhsG?jFeDYDoM|e@FSe|7q3(pDPAD$VSA&=8TcL`X$14BQQkJA(tz9n>%B;FWy zg-5l-%lW^A(ADyBnS97KqaiV1hkDA#`Fw=W3kkndJrHgmP7bx|Hv=HI(#+_DMhb@)WgZ) zNQ;(G!T8DfTbT=y$$6C$kmD!kLNdr|HVx`0WI!G(=54pK!LiBh8(7~Ur;7+FOf0B@ zL%S;i{tJ{vtl0O(uBbbqEFzgOI|h=v5)H+izN^7zBwQ*9gRF#Ctb~!0kYy$O!b+GZ z38Srq1y;h7l3)q<`cx|+C<&9SH*T~NQXrX?eXNA5tb`ttU?~uK4=Z7uB#g1%NU#zf zlZ1&@!k0`#qk*4Df@URrq!NY_B$GaCV@&!7t!~N#uoAuB?51TF7MCPM+qT3?cu*2< zvEG$9VZJ2fSP4ZcfsVD&vA-YVkiM>j<%_Rj!TviEBCcu$Ku1ClV+N@%MRhWeg( z77+D22Yr+Jy{r%orQ^z;kCrlwM5^#uH1WA;qQ^*_ZB^l~09Ia024m2lR%0fFwNVnH zG+JUMq+U!ybpG9KCEO+nKeSMdwi2F}ggdQ-fmTAjBwS@B^t2LANkY1n(AG*wN+%)8 zlJ-UWc{RDse!hZC^nf3}4owPO4KPzC87Xt3DTBf74xE6A z>CU~zKx@7yL`jD_ZX_M*cvN9xN}@#%eA<9L7=aOGdEl1@Y#K1#IrfZPZOyd{(vpkV zpN<*ga!SzcaXEe{IqtEVRA^Q1Wl6}l5?ofoE=hRMO31PjhC4{O$x1liN_bEb23iSc ztfBd(B#f{UWc@bk9gu`5F@Hz`!rXySM6~@((x+H2zHGhNmtE79*INlsSP7#gAvzHs zv=W|?1WbzNytz{)us}bt9vGebbJ7`v4(p|E#v;wpFoGdX4E_8gapBENAYORcetb|* zYWR7NS&5Vxy)D~Y9jHHYY>mV*;vPEgiA?bl{-La~!@Nt-QLg3kQEf_6DJK$|>ons= z@M+vvaO=#_0aZL_lD%yOGInNYdlVv86I-hKjU&FHPV{B?$*c@L97ss+OvNRN5(gvX z8#7AVnj{;K#?dxh$`x{OkDGqik3-Kzu}g8&?@9>08V7Pt^V~&Ey3S7El$;H{qZKNt zr)yf6@S+jS_uMyYrh3OC?_jn{s2SeJSYLm8>6LE3)WueH)}2^&b#C1=p^n zB#8x-tDcCBG=@*NV;0w1Ng$h3U6LH)IsWmpXSkh?a}GZtvrybW@E(qI@@nW^^{R)9 z3IndPm7<|0c>1|~Z;Efe%l8ZP5Nul;J?(iG?|r9lG@;#_^{Rws+}MlmaeKcfw=&3> zlm?{N#mb<eePHM&70zbfPUZ|?Jsd=f)n74T_?-O~u)qNtrA~C!*(k143 zJ_wX$KwT>0_Fb0UkSe;;cV5|v zOFYEeKIY->ozC3A-Zszsy~sIa<5m#=kdsJMx-STB|B!?7ya;4vgAYHcN2G2|7vUIs`$=lgY;CTNqu2!q`baBg_ zp3vIp)Ntp}PIO)goyggVE||}65IBErwAx`d>BpL39Ox?>WlLXW#p8>NKPRo>Y3?Z; zW?H29dk3N#6aKdcyk5?}a?bx_mO|V`Qo6EjClkaVrp0NAqdHN960OQyrteNAA(c6@$&wopMpA7uN?QLSzzp=fo?_;?@ zFw6Z#U3iE+Pb=~9yhK^yZbL?!ix?5*B7vd*r&1Qjpt|iaHl&Z7hEf-z=_(2jHxkH& zS^OY~juF;}01rfniG=l8A%o>h%a;h_z8h75{0CgbA%&aU!rXD5V8+hTB%F_*%v{S- zbg_Hz6s|*4kr**8LZ0>dlgvb6?=L8X7fP(VI|U;n9*H(YTwfzL5Xd}lg^lO zg1fy|_7LG6rfNQgqgM+Y>tk`OldUA&69&nhIk(CxpbVW7)`zQEs>F{tUB66`#s6(_e;J`Q{>A&t=-GcU z@^8-jN{~|bm$AvL_fP~#{0EuR$Y~F#Z{j~pij8k03xdo0_4`@$r~Yu2WQ|5NnngsT zSwu9t&XMn%Dg?oYifz#cB5e`g;s((FQ{e~hf!1ZDZHR4*wh`CJ7|rNaget7Xq?JYs zYI#E+W~>eR4aWS^f2Y1{^?nm$53pPDJ4{jRAG)9ICu;T~HJjq{FQwdnOFUyP>tbC0 zPscOzM1P9sg0j=iMcs(kpXwSsDs9eH=nWz#6rB?u>GU3uWqN2?Q*%)c(k?kh zrrFEHA%l~iTta*sPvHUZCsz69F`^!(IK9E_N=bFw^BM(DHE-LWR3Q74)mhc{auHxP z{~Y8fnj`Tp})-xK}!0TC@X}Q5k31sbH8>Rl4W- z7Wzn}w@%S%)q~dC%MVkS+&xD5W<}dcy~2~uhT5t@IM+B}&!|{1%C}eKA=^sbIin9^ zbVzVhiKsvSyJV0XYs}tv`R{Ad`&!N?_J{w*E_tgFNEKR)^a1aO9wV1k4{5flLZ!I! zFyx*KC_a`D>=j2y3W+n3!A@dhU0BVvN)Spj2`#jq7o+m1)4ULt_$5S$xYmr~tz|Z} zxGwo489gouBC0^Dx-+?~vCVQ|2?V5FP1J7IsC`7@{JzG02kIZUZ;jZ8pE?0p_i^H0 z$k*n|Jf`j!Cl(y>8xCq!Unc9v-VvS}ko#S~a8>P1=5mz*sW*C4m@7XZ@bw7dO;mgA zH~I$A$>+iytmdOq z!_FA`c6sv~kTLmU_0<~rZq&TF4TP*{q+JO$G`K97uKBfeIo#~dA$7Hd6lr>D4Rxs5Of1EyNlkXKoQHOz;~oTxveN!e|&{d69R?Xo9^}!goNv* z-HCfN@AQ@z2B;Q6#vg{)p}DO2BMahPZ=3eBh&-z#x9__Dlu`Syy+7c{o5lk0!{w~C zn`sI1fd08!GIr|Muw=;6vGTUqwN}=Le`&ovpE8t{%3N>zz9mET@3;^3KXZRTU#O*Z zyuADWW^~%qd|T^nu}~#FTdsO|r~aAxuGOdh;B4#df9igZxeN%Bp5ozCkkb&>xNj)j z;(m`=G>ll{=mFn$zsC*a;shVJe#PJKF`J%{g$e1hS>FTo{S)_lO!|^JTJ=V}epR~P zaPxT5@f@70^h5%j*#5{yI3Kd0NGzDBB&ABHMaCn6h|`j9wsWlM6w!;G;{^p79)XAh zoq`OEKQ_3Q|bKz98qW?)NzCZ8)OUZ6@9fg+~r#^vQE{ z-dA|2NK#UhNa|`Cz~KMwfQL`z%oKqZxey`w5VvyfuZVX@M$23Aal%^Pbnn z#CmQ7YuBX0BeB@JwJSWLuoy39xk_sjFjtF1T7lNg#keyZV_<`0wZYG-KH@s!_PENV z(vp>8vzEZB{zn{@^pIP8eZ-C@uCRC)t_e0~VlyNqcw$l&wpk2>+>S<`69m03aQVmO zM)3%e^(K<3`b~mC-YzlTmNz4>PgSpvaru8)LizC+>P!sN^-d8Cvsz=At}tj|sBDFy z!Nf2_{{Yj!s>9^g7-lF8d0&X!eS*uH55hSTCkr){io-FB&R(Qskc*+IOH4!`st?@_(g{QytNh>^) zO*~5k&sWg6YE*-IDW+qMpk@kBhV)D;JhMzZvMaCuA%dr_H6CNzp8j!2|1eqN`$uZ% z_8UfZePRTUSf}F#+Q^{llP>+!3eQp#kL>m9-6MD^TjMdD0jN(;>7Q13o-^@?K&F30 zvg)5#TjP-osy^Hs@yp6qcwRN}h`WIPu;4i{DTS?pD?0rh_bL0sZPm!7UcQN6!Hp9) zX!oufD~xj^p2^QTWU2&q-`V?FM=npP#-q?~y74GPm_^R#6=MnP9^d#6mgDsa6sUF95d=@(f1s{Zs4&alp+i482gb|)XkktYPbEo7u+{7 zAVXXkJT?9iM~NUhHC`41tLx~t;MWC&)Tis#7#$dFn!!~D_cusD_D%~iHx@o|)Yd2I z(tUwa`t#D@wu#)*H&1^a6U~kO376iY3cp1xQral{TT?F`BX@?5A+TTefpB-uyO#x1 zT{zR0o|cidmlqQR-f4g9l;9Wx4uWD`fdSQFx4&6gcxYilKfK|`%QcV5-DyEln}=4~ z-x|!6NOI`C$?mK?ki-sKFq&00Fi=%o;C>}tzLH)MNLNSj6czn?;#wMu03vkqbpbyL zsue)^Lh#gBa*SQe%?3|8>X<%(`l6h*yq#06K*ECn4Owe+4+%r6WBX`PdH+iAhI*h% zvhyXvPSn4a_7OrpVy_u{E0!9DzkdxP~8wn2{Dy=bYGu4ZoW!ZDa52Q|*Jz1)|&o&N5VoSZ1`_#-ey_gD6W z;EH~y1NAq*$VC+2@5k*!t`}=Z*k5TLk$A#YwyI5j&gXp5pqo2Nl%YlQrsgE)S&_Vkd6sm#CLOh!MEC6o)Zepw zmHm|i&cuDvi4fex>cT_CY2&)x#2V%FRy)h;+m|)l=A;Dc#!-juig=B2X*59={SLFg z*}n%^_XK`-&x?*{_&K9TBEJt%$Ik>dGpi~!mq}iqX!q}K7(aVEPj^#WH{{^Drf_X zY0o>-3c2&07!FjP`>VE$`y#mQj&06kRh%KYi=8{htec(9f0%V8_p8~<_d3Ov3i1=!+WBH#OZxqbNxuW0f#imgYjIZ@~Qt0b+Q2m|D>mn3- zRljPu1d9t4+Uw8wWW-S@;~^Bvq%k|aQHBVRdH)9_0e!yS??B*pcfQy^LZ7eqI~v?p z0L|X{A~Rc&O&iGsaIXMoSo9f}GeVznIV1ELmoq}2aXBOO8JDxCl=;unr?h7=?RiI9 zVbSL{?p8DCbGt#GLDA+SborIlMJ98-{Do$9)=8M_l@a~=|3mco;u(6UY2m+>K0DzR z^#9YRns5IAefANx^qc9^cKTcCbN2ts`Dcey|9<+sZ|VOF=<{$a_x|6>KP%86Ec)E1 z_@@X*@K4e0x=lYJ^jR6sj*%Y%RkQn@6an>A@GwFI^68gqAtSI_72@ey!vjQ#r$0qJ z4YOE^czReQ$|KIw-G)TD%8)3Jh(viL&%2}Q^u>W;V-9bZ)vSXvC+tZYxujeJL4?fYhbG}upJX@$DE0Jp7*FAtR8dtE5uVzheRYqd-*U#qhU*C zbVv*h;z@~~m581Ng1+ldgZn0IqOR4lu+M5jz@~(1XKB4rLbXxC;XHf(<~;9a5kIT7 z!-v7%wL@OcW>3PH|k0SnFScg2i(K_XK68$LWuKgVcQg-HCzv2hx)75R{FXI0A9OVBvy-ZqFab ziv1p>p1_+4!G((mu`kx_RI|Racaf&uyJaj!fZ|>lE#~R2SH+-#fhcca#?=Am!0l@T&f*`g5#zxz`DhLl z=D)BGSeMQL=Y4TmA_=3@?QdT7O~?bzyW(DIjdyfzz zSxaQXe^nA(cIE&g##h8`Yg2Y6$zE|ebU^L0UFq_9a|XNI>S=tI{JwLLv^`EA!B~>g zUTzqa4mhkjAS4~o^zZ2aL7((Z=>ONcK-$lqb{n$~LS;>6`(?u=O`JQ*0%O^OW{)Jv zVXyc-9s~HF6~vohmWJ*{XOH2pw|X~~HX*krETmWT(IooI8c^0R3X z&fEcK);P=)Ux=q+t~x0z!xz;j)sE5|%D}3&tlm9}MNYP?%9YL5*k9uA4lEyP<#_eY@=e1vA}0!`!s-*g zFTVY7v1+kIK7T~kjQdY<+4%#DY?#2fZHhIN_>HiR@^{BjuVCX?qJtJ5nr5qBlZf@} zgTg~U?vT1(YbG*cO|A-kC^v(YRr1_Sg{tr)KAQ5Yc-BneW zB<3r(J*jY4v5gYj6~5NfHnzHO!$lO4Qh5JDZ)zj&Vh%1`=CO@kTX^%_gh@N)ojkiB zEq7ExT7}Ds<)xb!CQSOkNJ~*^b&{4%8jJTilbVbq_#3Z@>02f3!ybIlNa?Cl#EPm? zHexm2VWe<>IPauKQmU~lZ!%JPs+25Ac})r`ymPc|(g`CiU8Utl(&`HnC)y%qVQ2

U%2d|gh>a@v`%JP2bH$5aM=|JllGWtoy|0xN~KRMe0N1=*_4BnojB9%9hfrZD#2)XxeJwM(e}kPQy-BteW**$`-4 zDmV{K>mbS96NCmOnS=~BYNfjlO*=_(_fU$#yJgsRvUv*?&;x z+r-@>{UV;R!Y5Ayq&Q&sHd8)eSRf9{ohRBR?59JWO4ljh6s7%x{V3AXg4;IR9z zVVQQmBX~lW@fgF;4U??g?N?6qSUsAnjN0xtYdc^`aP9cL!|W@2b_l-I?<7A0bmTgg zjX!@7sG5JpnzERQl}RW zQF0BPUx{7$14iRRO#4;#hx?E$Du%O}-J*=o`TIE31=!LN6&vm4eF;Kjc0^Cb?xXA7 zzuq4kOaE*87%;jzp+1LPXiLmgekG>1B)0QHd#cdJzIwc{d{@+ zyZfvDpPf&SFZwU+FEyRE5E-566h_C2@{?5nqx3RT_(`Cu-&q+7f|4+lpDp~B4mcdo zH5Z(3^2_30PX9B)Q1WuNdlrZL#52>~1~ZLjv6uf)7|M!%M_Muzi6i_i48@@sO4kk+ zH#rXg?Ujw^bl1I~{W#z6xX4+XchPjlJ;BApWO-0j)Of{u5di)yzgdSmgyef-HI|pF zmac@TniFd(cO?z8xi6huwJ*tAwINip>r9~j{^M)C)n)Z`i2CZ_e^4OdZb*5 zn=1;|pqs6+uUu2sbn$}jWlg(^FGLlpPA6mgd4G@#rg9UOUrebaE!eZXj*_D350`N1 zII?oul9YDgCDwiz73`F-fC?U-SSiZ!b;}4f&cR;<-Dys5-bFJwBg`dzJtFG|&SLn9 zA=RE_#lr0{Brl6?%lRw zUrGI$K;gZ|4>>FMCOJ#?W;ty&&XSrs`&*}_GQWL}9eDf7nt(HnZLDXEy?V8N%c@W% zojij^21<)lQ(lqJvFypC_*SwJzdE>oVx|1(cK??&Pd4J&kXL}I0+5Y(Hsk>mbQ=eN zha7$xJMsnH3KWRVc;Jz3cw=8acxXmtu<2CtV#%@W0e0m@;tTD@>m2$p_(?HdY}u8s zC$1!gOAt}Du)DeOV7G-YkJmSvE6W?HbM3YEUVH7e_px`aNV{>j{r2m#k9|lwm!qK~m_UV$Ig5^x;^oGW z(bDyvi0Mj*iAV=1(j78N91<&#m1Zo7rmIaz7gs`UA@n_6-63OE_dE8kdsx)0>|^gz zWe}RPXg@1rDQQ3KhRJE?p_^*HcRRjB2~sR>pF0S5%|Q>6N^Evyjlqe2SYv3^(ck7g z(1O7@#Z(Qm`KrtmQ_4PuHzM%p{WTJZWDX87(VB7~-g6{qP-Q zDGb4$kkmP-^$hSc8u%%Vgi8zF)^6y-t*HGqLt?$r`@K)&xW#)^+QCS zq#w#t_|`mx0`MJhcJd)l=!f!5?Q@tB_Y^QuM(PHLDr`+n>nwC+M(TVDAq(i^N{B-V zAq5PJbEguL&O(sS=q8cRCMDJ7N}P>Ksw^eMqlA2RSd>+c5IVpEt7$Sd?0p)4`1dOs zq4&XvMf#;H7Gs^lOqaQO0AF%=ltrfrXKDES+Ty|72%4*`pgn@VNvNtA6w6E)U~{dM z0^NwrKVr~wl}1W|?u|k{QB~8VOlDJt&!5uj(>x6Wn0EcUFv5IE#`}wOx=dI|rz16) zRMcupJgR3hPl@#`H%)>vs4(1P z-AAd-nWNIf$E{fCWNVnsDN@DfQ!et!==tOgRs)p8Q)W;~m=}wrE&Y;uMA9)m&nQWk zqUNMs=Q)f~k3_H*2Mjt+el8-c84Fh@IBbhCT zg}G{MRi+2_rt~b7`Ew~OO&c9G)?^ezbBBUDAQFWj z0l|1>&9s+UZk^#1jVDMko0fIOd+$in#}IE`Gf&hYRY7M;_=NWacm(rnaptg(?a$ns zW+gUA*P*A@je6T@e&2BsnUre28{J%<4zwe!+i)YpR;#S;hp zJp+})C~ypSp|-$_^~B4j)-5>ebsl#deD*a))z*2dBIM|y~g%CaNHhJ7{f(W*N2tuf70a3yXP6@?h#`SvX z7-iz;gE6YH8*rlYdz}2JQ|6&OsJd11Bla8E%=0S`n=vr_l?KlpyN1>)(BPM@3vZ;J zFL+D4{#2SI&1=s&io<%ayZ!6*2|d4fM7>)#VNlP%vVipZ%;BZ&#S=6TxRKv;*o}#> z50t&w0UxC8-b`dwyAj{iHK(THZ*eO2pTp>BtueS0lr$#8*l`SW@E+*km=T0v?C8bo z;k=TxPD^|70tuGFU7^k&-t;Cs`^f{t!$sYANjJ6}(Q@crHyy#c?TPnb`#thT1eK@6 zV7IAYwHg8GIzH3@JO0f!_3W)SeQB)~BQ#FvN%EK?@n z&?LZ20uD<8)DiIVB)}X34o?EiBj6QDfRhL~A_;H`0kuhh(+Q~R&Qt%hb4jkLBKCh0)JA#hbMtQCE!;ifj=wY zBa*=XCg9p6@D~I;Jqf%+z(*#5hXnk}B=BtlepM3qE&(5v1in|mM<;CvVFov4z>a=( zss?gGRccyc%ZIZt!pu5Wz~qN27H&>WD+%s|yr6AwjTczwaA_zqjVJqc_G+BF?QWl$`s?XA{*+)rQd#ATPDFIdpUq&EK1 zP!efVZ=jGkeZgcUB#%k0h%;UZNoOI@7rb99^4SF3vwXVuiF`IHsrr-k#3 zwZ4G9qCwwl<@x~9yK_b%v942y1jRcVfSM?Gx??2B)J2VO)iI z&wg6E;ggsm2q&(Oq2AC6WSa65uWywW21|dv?A!MMp_=h5@y%O&@Gq; z=dn7JgZu_Adz_MdCG67oKf;AN83rslhISIao!RrKl6o&seIezHl6tR5t?Ri*N!=mJzL0W8Nq$@; z&*`}V$w@h5ah{~EPO6)^+IBJBOvx*3asB^K-OSwGm#&-ncZea1Zf5C*fpjw~5TmCA zphVq_@x=@v8JmCD(2K73+5X?lGIsr-pIHmL3it-|{l6*DuGD0lg?a`{4WXVXVEP!S zV<=pc!L%{$mES}e^NaQUl`+ega%GIWzcQvC(P?f;ri@t!Wz1jyE6SK>aCEt9(hb;c zL`KP!F+7sYBPCPD@JM4RlIE{Tlra>ic+$Au4at--D*em4kAIB*B9m$;`&CAx4r~gr zi>#n0iCT<8pO$8BUv?GDsA*3i)yn2(sOx~dg)@ri^*r|TmLAi^?^m+^BxBfhVPZkk zG-Rq>KZxx+Kt*SE9%sSyp=;7j;yP7~bY<-|6Jbp~r<~q&MbPU^B=va5eL}U9iJ;XJ z2lf1!0_eDVCJeDtZ9k++V9RhRcKs$-<3zI1c9Q-YN}uwlQ=Sb;31q0 z>!q*CsOXhwp^N_DxH)#^-n6to--3EWR=igZztMO=6Mm!n1HcVO!)NZzwSRs`+KoHI zZy)=N<|6mU;j?$+Ohn)hSTU|1IgS!7O^c>N0#_PKdckCN=;V+tJ(^A_&6R%PbcHNk zM*nmf(RA;!YQBYPc2hO6a9us}F6`xXfI+to>Nq^81~OYQu)-L5?+kHl3oXsEF122DEIKlQly+dIn`eMm_|En@UE0la+=ut+$v{aG~J&($+mG zr0g{ZQO_n^zC0-tYP!r@Xe>eI{#&8v`e4>Eb0R%b^%esdd;_Ai*7yPyUD94NJ%v)= zs3?0&pzJ-SLC~aIj2)jsOeb4|cg%x9r&v0`;%pldD zekhXb?J7?S?6!pi(RsO_6P^(~#FLs{3|$_K;^^j7A=~e?72Zo`y|W(1y8S z4^f4s;7vv%ps9zq6s|uO7+NeJ)UNknJHyZsPrt|#`0B(8o$6V|@U_K=YL*Oz`Q>e@ zdS1txl%k4D=!(T%rHXr(f~{D_gsmiz6=cOC<5u}(k%u$PL~%78xUvLa@BJ~_hTQ_1 zf#Od@B<#=hF2{%ARrC_So-G3~em(2#VDkw-B9XYB<@bGoEtP$TPp+b=j$ihLc}L84 z_&5ojkN8msD#vCZLkB%%HuHUNA)EP>`@k?44pl-nB0l&pBXzOsh&)LDzz^vQ1&>wS!7B*lJ;EtX^~UEz5kvygip$y~}kk7l-z=k3PN zHGn_M?h&znv0ImQ=tyYYy=yaX1n3K_G7)l^h1A?AhtDWL-Dll|BEhbTYacAaC$q!M!$+S*!N@cy5nQK zUjG4m#i@^-r;+^^lsxNm78X6uvHr#upocYQ6R$k1&bf!*ZUWGw{0pSi991i7ZY=EI zjHRp@iPctDQ&i8sKUOXKepPigp*qoP&CHpyDbh*Y8$BDc4x$H^m7suaCQLE1X_^`1 z`TFNa((;fU6OmDUIwnG2;uOb3n6X*7@3W3$OS%0Ix9=i*Q93}fQbz{&bT!W?oa}hy zg+u7-a8DJ#%*lQtH*NDaob_8}-E42XWoY`2q(?(uh6Mp^`>^vLUYU(73i9MkzVA7BOHt%PPm36ln#u1z(a`^!gT!PN`B&r9UFOz z!-yIfEM0+>Nt?SktVMyL&U8|W9W0?{pd_LQq+6Xi958MAUKY4i2^^*hOh+JIM~FBQ zf4xGy9tVd5iW;*V4)Iqsi@omUufxP^2Y;oi(7BaeDP%)hucdWc`Xnt&L9-9?Llm-f zxaS|J<(YfBuhsoTmKHqMAMg>>st$SB|0eQZHYUtnb4e8W91d&7TR)DbdXdDDBL&DTD$ zC1kt_){RBiKCv(Cg}i+w?%K&;^hU^j?Gp!qltsG;v?GRxzo_uAneea>TYAm+?o8H< zuLT}h-8l}7?Z1#=H;$7o^}t0D4)hprTKIYh){Y0FH_MzA$asOC1{}UQEAn}Pa9~`A zDG)a&wLh83u@s^VVd|bUtiDd!A=03UH}Z zpx_lU%T3)vL7M|Jw&8&C41P5o|GQl=TGs!aif&#@<-?5RH@KS#9T~wV;KHl!`6y)6 z-AS~~Iua(=$L4Jm{dZ>3Ay9QFWURR9i_qqwpz42x1uHy9(0%*D6*#bS)1t#6`_Ly2 zQO~|cz5}uDBQ8?N_~GxO`yO6&7&LH{XyE8~*Lzx}_lCp6dv4Mo(Q{qYbM)2oQ0H|x zBK1d{K0?P~y50%*mbUdO0&QNC5x&F@RQm0L-F5GC(eAqLo-UlGX5Ce8QM!wr<&E*3 z8Rx4wza{q97Nx%qGj^l?V(d1Qv)dl(uU-Ey^j9@BQ87+ifH%fzSg)iIt$Q%cFJ!~~nb;{B z8qwB_6*I)@@#-ToD?#nYZszi>@p(QE@QLIN^8)&mBWCavI>2q z{ZSr3sNoZRsK!>D&%t#+}ADg}by2dHwrBjnvGg?BoA+q|$)7Qa)>Fd86U*$Mi!x=L< zeUbS9O<4*2crkpvF!%opd_DZ#(-&jJ@zBQrQMg!D(O*QztKk5};b#=atIe4Guf+X# z0}NM!o*tcZ0sQnC*B#~;^z^ae3iQ-R0@!L@Njp9_(h(^s*son5Pfs}TMuIPkx8|S` zr`|~Ljm4(%D%5%900=T%m;tY34A_a`YB(FNqGMGbxDSIdn@4)bU<*t=K9M+8_%zJ~ zt8g6-yQ_hMep!NMpMkwCwmsf((h)nevnEp}eFQGFU4J7jOi27IhB|>R&y^rezyk|< zS^`}_6oi)L5k>JN&ad%fYV2zAKSJM(NH7crt1XeGCINuF8e{ii3B*qQ6zYlBxR9gLfCIG9oS< zoeq%@9`%vCR2FohBX@4Zz_G6NV`%N%OZSsP<=FooTy=lM?T;M<4rqTYkM_sX6s6&L z7{RbRhp+=8G=^!o*!jpB-NGx@{2Af|`9?P>zUvzmU)FN?=pK?h`{DDtwc;cIc^_fl z&}5;HW_nO6^J47{FKVZn|49*PGC2x2U1Y4ZG(Vf9vkX7S*Xi@IW5VhBDIr zbU3ZM$G-nhX<47)-rMZPeVTA*2ULFi!(DQ^1&4BKc(9Z z7Tx*n^u8m7atl)#UGxWmeOJH2(nAH9LRs#icu>v38zN{pCEV=VOv(1s>wZ>=eJPnS>*Ebn3eGnyjlM=nj5_R)L(qPK_Acf>o zQ}8gz!)3NJml3K)8F?BOq#I22>SE~WBITvbp{bPF>y+66RtRELBZmx4w8r~@gGuQu zwhkAK02N;lUSFd$uk!LTST1NED*@Gc6<-Rf%G9w$>6cMfU8w16p&w++qtXHMXCgxs zlQ)C~NrTuU4cR~Jbh6o9YQ&og4f9>Lx=nBJsEnh z@9I-2;Yo|O3rJ_*y~jga`mR1f8-JrR!cQ(bfyKS6i)D#!r&7`fgXR?(NWj6v#QK5>%eKk-#NnH8A4TG3gIpSmg1ZU6teKsj9G9+5og0w#BA7Q301Et zcwI2t0^RveA|>!Eay!TQ49wO+6t@-m_FcVNFk6_?Aqcsjr&7BD1$Q_tMPWuY!sWA4V64b%(k1-K!-?~ zb7n(qp1$wu-GbRD5ofj?sOcWY_EBb|ih|j|bHr?T2m)n%)}`V#x-kw6f7wW&Jmow>;trWfE z&Peb0v_$zKWc1>igCb1g7z(d}vWSBO9ErQw`~UE$1yZ-&P#JQ5zY@S|`})&7uO zU+}8Qxh<3M<%N0CBO`ztDV!-lB5XK?jfoE%MPZ}j!^Tk9q(wh(;m==f;?LiA@aLZo z(KA^!L|K^>C!zct3QH(|5``s{Kb^u*J{x!^E&7#=X5sz!h5Y&FrS!Z=`51Qlm2V!P z{Hg)UUo6UR;pP2mlPJGKlz-?F%C8xq{Q3dP_YYA1YEeF0Fi`tx;Xv)@&p$7v=SAAT zet_~H7@+(|1}OhgQGN?A?^l~d`5mJCLzhtg;{%lcD*i0Joa5wX-Y-#j&KUkaT2zjV;Kfv>|?pyaB(EOZACrL9_oQo6H9agf1w zMpKMquu%^U41z4CfhMrC9{@Rg zzXNg>j_Chp%~;&JN{=e-aC$7(j-p46b`0dr?QkT>(!!C1WW1ALOAEq)_;d$5dy!|}y)t8cYi^zME$h$-2eMsc}fOfrT!6RCdFDUCr zwQLxI?{O`YB`%Kc2PKjzs15x2SRsG@ZYiFW=L6cuqvens!OLNT4J(Jq43|(&izsK4 zD5pb|b0}I)C|VB53%ndQq_A?BY;XzXNF*ard-(IQLjL^SQapKkjz-HNxqz3$1`bvZ z$p9DWpAhnYutk)!NtDyUpFcW;r#bjC8O){9vcOcZPrLqWWNLP9r*~5GYqVwUk7!9? zFt`7RR#WDJ*R-uKVs%y4UZvGo3SdtD6yC5~5U1b7My6Tn3w2i9L`KzcBS)|xT=^dh zDuZD*qJ13~md3*H=_*yc;VkT2Tv$2_J0BM|iiJV%uS%PNu;I|JsDj4ek}F&_(veoW z?zDua@$W75ej&yVx}TzV8up4*>%Zvtj&XhcQe3q}26*ctEb#*oWKjxp`rzxGpQYiZ z@Cvi@L&6{3nns5#HtEq%?DG1U{vG`SGkdas)8sN(IuyWb&y(`^*S9M-UtsG2IuUm0 zL1vLiOFXNmg+;&@_VS=La?k(Y&=5h>+ULIhR&0NSPjv6Zt!Oy#z9tX0GWWWzFT(0_ z|Df=zjr%ne*cIO0xPJ&4f3ExmqW`J8J9M8zvp81+FP7|qDbNnxw?tEun~4B?zGs1^ zKDRjZ-Z^RerwBM1y3eBV=hl$5=t86q6_rR^$V_y(c-a;H$5-9<&nN3D-a@e6Pr;i@ zr1$YM`F`37@EPw_pggfeYga%F4JdnE{bZGy)L|5@n zsMFH8cL=K;>h*Z;9Xa8n?vt4r;=^kLVObUYV4fQbL3zg@?9ELGtm5`rl&%pC)`P{(9;O; zrJA1VWAqgM`pQ%zu*gY9MW3K2*tQCKQa*Api^!>BKDKk(=6+e)HvcqKJFe|8_l6>F zZ@AC%#sa!m&-in~^pdoDchI$a+o7+qjYC`dJYDGk*AOb4bKkKe|d>Jhqn z?`kBax4LKPuloS*{L{IwDExO)$-J+7_XNL|FY+^7%V+=MeO*?!7FNkHD-Jgv7!-D# z+YxE zop!@Nblshyq8XY{XtPCg(r);vuDjR%UW0DRX6X1vYd&L@8r{Y!^|r42NNDpC&3o9Z z&~={yWopib-|pLPfBG*1@&R5yus{7LUH9jd->Fb%Ngwp9mcFl8KG#=MKKW~8Q2jr? z+RnnV+)7x;{`m}D#VagS-%O#;Am+L7KZpVU()j8iv~mC7@NXMm)r5Zueq;o>3;g-& z5ZHOUUtm;E4SkYRIk>Lt)A0L^`%@|KSw`aIdx4QRD`s*&74sVu^RhsNeTHVM-GBz{ z3l-@#ui6d2)O8=mDJZ#OPiXV?z!18o8w}ylyla2@RbBT-p(6`4pMe4UdKelQRxs9< z^qsLAeyhVrBY@nO!~f_*N(Fx(+VdN96266s=I*vX1OLas8~A!uC8+PfsirpAbbum<&j)k?a})ygMnU515oZl-Hx&SFx2s2)b#M- zuSkvDydA^^TEzH$EE|pfr83}SrdT3PKJ`_xdp^P>91$$QK6=C@WgB2iSCp3 zE|Ac6nq{8k+|ZfT{UyCV%UFVq=G*L_>p^#{E?GHTd-<6eG_zgER7uxAU!Uoflx~L;6l=$U%L-i$l(%NgmE9HhJE4Bp%FTBHSCLF z>KU6RJaKd!wZBl)GtCzYJ#R^%XW9*Bb-;WAdXA>&^8=*NFM?(m=RaDPM<)t zL{2HuFYzazTIeoUT)H(H!L!?N-PY_j^Mth7ZB-MJP%LPb(QH489r{|7W=UH48O_;( zbT>Owex^|Bc6R5hLa*yE2~5B>c54c-uL82|rJaf9?KDREC*)!J_yXrQzv?0ZV&Ip_d8sPe(-P*Ikq zteE9m!^Xrtl$Ftnb?^F*PES&McY|NL<;Dnq%Sv zaJRh@?b(AGU$1#_Jvx_h&#uts$(r9iP9hFa^6UhnMa(alWWaji(YW%p66bv0!5;ep z>56?+3sgtn0}sOspQUJCVc4MDi+I`hb(z*!~764_-00uX~xqCa?|BDU+V6Iop6-iEN2kk`DAA zE$k4oRQzC1^17NoXuMa;Q&+~xQ|q?wMcFZt3bnpA^{amaYUrdF@%go>Kl*`q&5PE%f5F<{4GVgUi#4Il1`FS~PVeBIz0d>#-%7DN3 zkbM(??v3`5n7$9Pd~l-)b>dGz79`G01~uD9Lgv{{gy_%J*slVucLJhsgvAHKR_A8g z*HHy>iz(q%yK!#pG-E)>U^a+(qo1QjZr011J?!u6_TRvN(0+1ua6WetCvFFk(pwl1p&@z2Rw+drmui-iYJ!Uv7aY*Fy;j2*2tfS!+lt!V7A1XWsUrSSg!y}IMg1{Rlq1*El}-b6BQ;dMW6yF zLZC{=T-AjxRL@npmnKss%tcY-Wt<0z*;kd54OQ@KQ}9&JT!=U(C=}Y}rJIIsGj;aJ z{$Zxzaa^Q*>cp`jw&~Yd3Qk+JbGDWiyu#1vzGBq=JY-D!IPkh9*cUv1Y;WuNh<1HF$~0A< z=U->il(iLKE|<2CO1mvsb-8?7w)t{hz!2FMoSjkd@){j3p}aOvEgz;!IOZ2VJ+kn) zsd`=pOg+*~1+T3B&=UO7HdNl!)|z+4Ajx#|RnXL$v~z)hpRj<|P+D}xU$TXNll82* z{qBYRnj3!~N9$G%T3BckVv(jLjicSL1bx&_rS$byPgW?~7y;S*QdJ3JKY-Z<*A~1~i04ORz~cW#ejotBy{s3aYusXjRp(l5SIY3Am9+FU2dEi3*y6TTEMVEie!mT!~70 zcqL_C$=}@E7kRmL3SLsSwie()zpbs)72j-z*ykyE>ALguXd3o6dQjC)qA5znPsA#I zl2`nssr^2RX$sDkOS4}$2meK93Ld=M$f_N0583+*eV{Cc{nm1gh8 zgn0?gj?qIx`!A*ZR?u5Q`B3d}v|Ii!luw&=3FU86%HK`qRF^=4Q4!(Wv>$~fl>Y~% zd<)7qy%c4EBlzl&sq?epX6IHSVN>vRb2i=YsK$S1Y3E_k|0u16=;mv>QoI13K6s3g z4)6>309WAjf)BQHKG;@Ta5@0#CH0zOnv|wsQW`xDxwnrVazC4V-obgv=dj+y4EE>6<6_oQM~)K>FxT4~#}nJp-YzQyY|1>dFOt{p-{)w7wL7f$*YBfLi=0y(9g z^v|Wh&yWqpSjYn zu)~CFCQbA?8*{Ke1x>C>(M7ffn=^19V`K|%1RJ7VPv?b8+a7s~*B%I`^Ax5S=^Xj` zB(mulj!j_Rb$d8s6_vN>v-M-BKQRp0 z2UnPZyr2kfH;?em!#?-Voa)-w#SX(+_3cdoSUN=w^axNdCvbtYWaB>R8J zHZZd=jG2RVth`{EO_&Xb<~#w5;lH zb3v}QmCn?%4|2Vnj^w6-t%084>D~e)rVFIfEy26ejFH_&py|Q07@*84ZRqv29BW_rG*ptON6tB3ip!VKXVkW1=IjpMyvHaV5W=?te}LYca#=%tUbg= z&9>$Y!c=Y2NG-_y`hCoK!3vHxr1mDRzJfs zG5dUV1Fe2&eR4--xvBL<;%o_{I9ngwv!!iLd27e{*3$;<{r^I=+WI?0KjCt$=VGm{H+jgh1w0Y)&@+6H=TCF z`~1y>Hv(Hhg>}Sywhr*twnnb?+j^%H>Uod!hVU6EQ`?Ui_S#ygEyu>;{m1Z_ z#QT-;aMR#9RVJZC5Z#IiC02@Wg8F3$o1KQE;J_QI=_nT6yN@chD|FKI7x^OoQL6`{4`n3g5O-g`dPanpI+iy&*=RN z?f1_Z?=-j1cyB@qh`%)WnsGsC@GE1LF?e=?sa<~*!IkFhb672=_5ArdOkADk2K#R? zbc^$C=HTb%;HSmG4~>yg@8U3IimBl3z(=f?mKYZsmxwiLRGy&nwxE;fNqJICdjIF| zpBVTj2L6eGe`4UD82Bd!{)vHqV&I<`_%1MzB0DbGvt$@!V9nd)KZR3Ow-{@Y(7dQ{^n5Y?)tNVz!nWt4#9UV!|Qc>r^*4B&0g=2{cgG5ZL5(R+%=B+VN!$5=~Bn? zc-;*iKVnfdSDmcOs=>>0hs&2EH`rFnSvA?iq+!F5M4l?w+MM+cg3FE-He@E>?Qqq| z?poRJSm{S~SQ)+vw-!yPt81v&v)5cZ3JkcGyWCAKX@#TOkGf6uI(!cA3P<$aYO#79 zHngSM>U06*7kHR;Q;~VrRpW03ep+Vzp!;q!8#{E8@N=Nl^1d)~vQULhetWrBDHsu9(+{`Tvfq{BiM<>4yGlH<*+0@x7I6MJK1+H1^+PLw>Cd9?O1Sc*exYCDlZRgDZ_6zAm910B zK`}I|>BY6Pns(isPyI<@N}kj|gnPm_^%3Eha7z6|eMJ33eM0>}eL#38ToaxNw}el^ zA>oa1Mff3{5dI10glEDn;gfJkcq3dX{j1Q0(yvN?A`1H{?N;bWp%aeP@a7g$e z94d6I(62(b3cV_PqVR{p7YaWp{e!I7Pw9umcO^XWSu~?BatQ8uWgS*_eI{B^X=S_;=V{R+%2$l+c38;@@u#Z)3Ny=_s=R?mEOLw{dr0#DcnA$Q?r*JJPhl{RMga`6XCm?BUO?2m2zQz&Y@K z)f>ngt_ObeTYZtcgnJv$%ivPQll&KR6j$xscqgZ%qx=T=%=+r$}F96Wm_7<8UvupNV_`ch5s-B9Foye3-j8 z;s5@|Gm*t`6t?Q$&O}=9d=AgEa5Le*@#ozA3{P@3Do3zwaMS+%OyrGUpNaSmo{6-= z{ZF{3;kLoO3wILkiZ{+gu7}gZ&49DOErDx*TLae)_e;2E;X-f+;Eur!dh<-=TDUwo z%Aef#-;Q^40lNooHQWPmkHS3(_X6C0xcA_`fE(O>CUOjoI+u>|*kHBq)I}bPF zE#wWC3s(fU1a1}FV{k9SeFS$k%9r78g?k3>=K1F$GjBf^F;tz4l-+SI^31|>k)?~y zMXtZ=TqJk#xrk&t7uk3Zp5mvB-^#qH$XSX!rNmR7iae#rP2&VWC!ST}=@QS4;`x+# zl7%My$UK^U%6y=duhfS_Ih^9i!YSd(yYgfiQN#qNeidmcDCtiDDo>?e%2TOd;!}xF z^MB%xV3b~wNfbEEt%@8qkbFe(D6Nt=$zhqQ_$hE9W++)obLU#jR%2ylNktW{qNZlm&|{cH>yzAS$th2lS(K_pAm3P|1+@=Q5TzVSv`7jXHUbuLFuHo~UMKBRZo>O408vg`qKsrCdWz#)%|l%d4J zGOn6MUGDI@9Q6}e{q44To2%OCs+01tR@2MWXsg@)LoC9k%Km1LL!N>4T@Ds-wLXWR zURDKiSmhMI3+p=aduRbTRpx1Zz=B-ucG02_ps5_2R*%=|@{8>GLR7EU%d|kmbC}e` z3LHD5AHJ*oEA=xQ91T``ptjcGl@Xy4!)KeBB@>o_+n6mXNvVYna=5%Mn@0g)z3wv|p_o;(wD zLoCYQ#Bg9SF0>YxS}Iu_LtGr(mBza)E>3=2ocsafOpJ>&alklvME+D2gQ%W(&5wWO zl{YAnIW+ppq4^4QBEz}I;cW=`3Fp)UvF;y$&LtB2R^aairP z>gBp@xz_EKXD_q)=bA0D%k7t)4IWN%P)s#D{Ns4v$La!-Q z;c1D(6s}pGBR8s8ObNEqbUSHNm1>=+$FKUREB>U&A+@9 z5!mGRF1Oa${5DlAn{T=AZs!tK8E-Y!Gf&FW8*cH*SyNV$^0Ura9Sy(?zQW=4Io&R) z-eIftvC3fP;_>SP!z9vj;+>M~4f&MG0FLFJg zz3~xiosb#$Ob#yd)(I(vNQHTaNnShWRGtY&fc}|OgI@d&HjF8<3YTGGI&7jSRv{|h z=|W#Sp(ZC1ks&_W9o`(d4g*Y99+8eZj-VP5i^E9AjuD8u3o_MIc|2ife2$zou|}?O zHb9%6Rj?``+nfCkh@X_3NWq$Q@e~T1k;KM?Ib=LlzFW?kTn`B@_IHWs7L4(Tku5`t zB&6M%JF!-tAf8m4CTQCS01U%`%t%gP8d77wf>6X1v;h-Jt<~@L+v=_GTHPLNz;`#b z8pBw$pkY$2N2+nMj*{of|AMN^E=LoDkL8Hv_XZr(D2U!`oL&T(D-h63+Zs($o!8+2 zZ#wIi`R(-qht#yp>31OZxGc{&G~MTK#{R|(xz<^aU5RPzS?#XJ{DAU((_}WF!S8ic zyS;$d+w6||8RLL_;KrA|TEbSx-B}EM^)*Z6)$;U}Gqb?|s!(874Wc-8h%;`QjNJ~8 z)3yA+TpN@r&7NOYS%rR8)kZ>5Iw>pH;E=$vfJtjO)zp9@^;I71)Swx9H~!|*3#MDa zU-T*@a?Tz#fd-Ei$t1VOugC1!Wb@YKD(^I@Mgx3)n-}yh-#%}WHIEOT>L5d6(B%HH zCR_W(n$ka(b)FcSlQ5p(xw^VQEn9?HYq3jZML#>xdFnBRvG6kRvBL{Wv(@_`KHMme zQ<#~A73%MPNtQdB_2Bn_L-yF5UcrzFBLg3SFfi1*flr6a;kDK0VvMMBk$}r`6;Z}# z`NoUEfHuPvX%Srv2gt<&v?un4Cc} z&Ymq*E+~>L3(Qh+=^aw#?G@yzB=h{a09eRbE9T!>DOpO4Wi-%3?v*M^%1ew@;$bZ^ z&YmY#mYNrEUxlU8S~kDJA}Jmoj1sICB$~gV%8CulVySYr(Oe=`mQ+cz%NG$7P==Kz zBPPr{N=xpP%1bIN3#v+_stRLSW%>Nd5@~Kp6|alp;W-zV8t0<&#<|G9a-OxcY>w*f zu9AxRQsw+PRRAuGdPGg31U^Ud_mo_L>Q zHddOf#;U3^;hi%_6~^u0W%R_dFj zl{djnfV(GOI`ZTHxl*ck*Xi)#%sv0HlALup{hi>FFacv9m>^=+8TE0A9IrlSeIWYc zZSX-N^fs^*!XE`LylD{%ULiH>7tiFC%SL4u&uo@xg~riAwGAlA5mD_6n``yT5?dN*wO9k;icjcYXFqS+1@8bCyX^?=CH zA!pr^n_nwiiZB4Og&xb%QG;HgGFf6;<2hXJK;1Ig=do48%E2Z3mpNo#wb$maUM9+s z3D2_5<#3P`#`IXW6|h!dCJ=a43V`fk<>RWA7LKt6A17{3S1Jt(){$Zjq-Z!0;<44B z6zX6_(W2IH$kY3&JrHfcrE_!<7E2Gc2{aM|t4*rWc{F)IcmZml6T`dWb4B$^Q3V=S zO|%}Osg~)O%B)O6r*TkTy;@elv~W|_2_%gA?vB2RrJY{pD+Z3ZSY#n7s`MdcFjA>x zt0^l)mc)GeMWo^=B1D|L*dmn|!{4JCJIFFvg}WW1}+<5QVNeCw4_@7d;ZKCQ8I#kj!0qM+VxFBIu1)y0S@B0>{%@wo> zs#BG2^=4AxTEPTZPD1IMM2iZ6&;_mWRN2Fq-BxIaZD4HZzMHKs4=G`oAy7<}jw}_u zUyTpyS~NF-gbCFHBALa9(8l8T?;Z@(am0b>f-+cwm^PHNF7-N8*i;xvf-FvghJ%1P zG8HSi?5rz=l2~YG5ja3-@|>v1ODQZ`^C&~IO@}#<%Hr$6ic(a`3AKrXQOPl$i0xz0 zu%e`kg;ti{RdP2m&=LR*12mCpfw2ofiom^;5ww{UIx)&YjR70f!Po-P5iWTRFu^K4 zL+y(JqN&6<%FrbVX1bY%Y+jcy!gJ*}*W2ck(kd!jB$zFr2|Jx;(~YtiuKKFzl&7z@Xr~XqU&v zQfPXr9Ui|Gst~I|mu$q1kHs6#iisN+;$yLnW03~rb5&6T!YTQMM+#1`0&4HEPVnhL;%C5V;w|tkJ*sFwnEkcM(JTG`|0)`X@enU$(J5s6tkAx-swGnqq_L(LbtT9`LVf+AB2t>%PD zaYb(28;#2egaT_=SXp#{6LT6P!~UvtEJW%PoZ%KIfuSVr|JMUWYd$nlT)jD!w0kp| zrjxvPLl2H6FQ;0_|4?oUViStaaf)V=R$5s7I~-CCmYfa<*ly^ylV#m6IYTi< zBGI~92udFHZY2~MM(mB=X{;zKEt@-4o^WC71JZywm@zsu)&6C8TW0ggcIb8?jWRoM zwkX7U47xa()XGpFz;MAK%>bLstuU6Gz&JDL?UH_qEeL>fjb0O2;>5{wtq#_K6c+7JE72rv--E5uA`-OR#sJO2cL4^y^q#Tl+h}Tjt}%%7XCc%fG=k z>$(1J+yG(fc#|#ySI4~+{~YA^$Oq>l55m#&e&PS?Q65ImXAt%z96f(4{QvuX9!5`U z6SbY;68NM(VYEYl#2-7;!RaXughZ51`H}ydIO;}nAR*S*vAkLk5~Bl9BYUTK{MW%A z3Vq;(C5@3>3ww{z4S$d2&rIE;bcXcIR63SFH?>yf&rHQ)`7={^HC?hJ4rP`8TlgZ< zV6ayDu_ed!E~RCa)+yFf*9x38@s?KlC;J0XjF;YampQ+n9=bIftr{!)90Ya$hc+J$0m5Q|}uQ zy(x#NR20a2FMfY&`3h^bt$G+GJae4T@Q> zOpU<4HTJUfGPu#^b6}U)>6deJbCoKxgwP**ZLux6C@&-+jO|BKcD#~yppSO&_zpV{ zgH>d}L!0-|*)o{eS?5qMH54L)t=_ZDCj2ClP{jcO26#N`843gQA$O=0>tjtBCgoCX zS=Ge3IM%}GrX0fnSM(`$Vn@}40RqPdRL*Kl@ra5RoF|rPn~ob*j?bzQZC1BWiRQ%a zA=#XuX}~IBl0$e!jr>uhuCPeScNw7#uXP7tAb~o?8q8}H)!PMpgb!O>rDY;1gefE< zh#B3{j2&_x_9iJysL{O?r5>uj9611l+GK1!$_TBUn6E04XQ$Uw3-nYG-WX#teH6tj zN(LlObGoW=5)s=B?9d{!<0Qigjx5@-hbV1ByJ=5#8pL6T-0Tj>z5qOan1gY_QXyq@ zFWHGPVUtnBtYK1-Q6lPbDEKZiG65ITvDXTPKFAm*N7WAghQHOGW=XEWRvOACZfo%M z+oomg5Qk+o;|;_tg4H=s4j7@y>!jg@l2 z7(b!)ocnwZA0unxV~z#tRXSG+S-x4EDOJW1cES|7CXRD5|FN;wkIpAPpTd!^B;{tt z0HQT8)<7BT2NO2hT2DwsCNGm~WG@&Iv2;paw8-E!f`JFipC5T;;Xas)xRfnsN z3^V-|4l7AtwC#d`*w|yn2EWldx1xj_{hN#xFjuIOqN8Wb4x}OAVs;u_u;f%7PxXwE zW8@yq!{+k^h(hIC^NPfrM1v#_OS|N`w*yHuwJ9XcM*|}Igt?T~!G6}2h%T&DT-FuX zjkQxK0%4>`Ea0+YEZ}{v4g`DGD9NKS$qc1mAk!>Rs5&_XS|NJG1=ib|(K!h*z355y zQzx%$v?T_5$uhv|@PHc;6V^>i^=oT+R8>*6UPmK#qcH{XxMH?v!E7L>V3;~l`)k?8 zV6+C{bY-w+xaD#aq63F)PGA}aOn5$+=3oQ^^>{IrD58kY=Wxi&{CVydxU6%#y zxtN@AsNLbMv$@H-G%~=1?M(34ivb2puiOa1-hew zwY6l&=xvAvV8h-DoPlv_a+?fE!t;}aPgKH*2C&TuL8dX_^n%TZHiV2U>u!$uY2e2= z;&)&!A5VxIFYDZpEEJPYCM3c3kk1a7p`y|d6FxvKSSolFT4Ac`S{Vw~{+SFA^@n~I z)dP{`2b^T;>L@H<*6}db**t8btgA;gX_t~LHA&&z;P(0Jn`PQC6yDp(AS~uLsQuWB zabp5i$0nmOB{tcPDSk2`Q~YE^rubn?hNVCy8Dk`bSsXRaKtn>Hna1sC7;Mn`CFWrX ziD9SKFCq_1h)A|0R;8ra20)A(G z?lM>?L6}iada#%2a%mYi38PoceEh7C1W6F<4sPF0`$C-C$c~&K15OfbihzxYo_1Vh zaf(T2tEDv+`gMg9$7axP6pdyC#N=kc*=*j(@<~-*I#@fP zK2Q%?2S%SDRj5t9Znr-h>oL{rSL3L)VNAC|oWyA>2}5R72QhUjW=DuK&`1QT{jgC4 zDG}C~tuvQJX$_14lvZZ6_{8-fU>}(7K$c=cxQ|c7u++w}F$l}`{#>qEVbFwE{Y)|_ zr88g3K`*imEGa9t&MsfTN9yPmB2gnCZYjhBc1$*6NgTI^miTgdxp=OSl(iQh(Rl6I z6hM>e-C2`X1t!p+CD5tEv>i6gFiVE+SsZyIKPL%{5>v=B+EPT9oK*2u2X#5D+iPUH z{e-PF2xT?S6;2;poY8w^b9x-iz6V~?w|z}R<_DYL@eYWr}#0m zA}aCkrN&*LS}Ti;7;QSRZVHe#%7LcS_{a9D6G zqVwxi*=Jipjp4SB)sJu^9=<^e7NLg!Mwe5!1?e-em;kOKO{Sty5Jh1# zNm$Ha50O~|vthhA=QV)>~MJ;O>e zWYFeQlb7x~VwR@UWMyarCx{atJ{s0s4z>p73u;5cK&MO*;5csF1?zJRp=={d7~<3D z&cGN7^#(~dR;&~;sgPWrP3sa}ENm6oldZmKtorK7bp;6`^Y6-MJ2 zFgk;_m|gKooOb0YC-Rimo1-E5v0UKg(MY^h{Cs*ym=VP^Nu!OGR*DH@0nUzHXrzGj zH+8DbpOB|7V8;}j5QqZ-71^NjPKQ`A8L99oz$OzkmzD z4Lx%{@&TUcI>~yGAC9(P@Ryi8G4epi`*CDTn&5YPaFnU8d4ghs3DE=Nn@`f?za9(J z8cFYmPaKSdkE^OAy(8+^GqEY*RZr5q5}U`+d)<=07J(!MNLiBM5{J6`Q%l&)Z5xx+yfuU>{_ z3+mngKcAZPgdHf^ZPX%Cjf;msj1#YHhtXC|h2br_n2(2`$Gkq&4I(82a5*1dcGd_4 z;{XV2euVhgL211cU9D%Q!RSS_}lYRxZLM46uitnf}LLst+08JTVHH6D7Sy?VOnuN^>bR6k8@T3b% z@DK-r^?JyicHB-8zO8o3;>3Lq?uUyKc6rG%rVL>n~$ zGUj2EM*u4~v zJ_VN!NFyH5o{tMZ@aY!2a^%Ih&R{Vt^cUk&g2m(D77H5Z72%SgJcllykcC3c&6ROV z#!PR{2%wf|ohj12Fx*n%gHE0fkS)oaRBtCNSQNs7e%iJIl~Z*)ZSTj>0L7zH0a~$R zbZ2DNzx@;vKk6}ljvs?S0B#%`#0 zZa}CG0T1jza7K}4aAJwL>4Rk{i~eSHw9(B6m|vLgg?EHV7@b#Wo)HQ8#3L8YuO!sO z5QpIoplV3uAXKW~Qfj-HrS6GW zL5ZJzE1Wpo6E zF=LLzHs|1Y*o>$WTa|}rfIP58M+PAYdE^>ub7VvAK>5_7k_2O=_qN_29Oh1 zWUxL_1YdRaLFj6n2(kx-Go7?RwJ1s~rFmJ=%s4PQ=dgK{m#-qISFe_EP{L-?(g}5x!Zzikd(moRJHLIN2gJ3DF*9V^9KFiPtLJ z=QJ-zE(=uGI|3NNtBBg>5Vg@XXci<=CCG=HH+g+*J4&peX*xxzw2?G$5wLhtadToH z)5xY&IEPM4Mc?H7Tp=o{6(Z5bDv9W_BX&?2lbGM3<_#PdRu_#95;UYBPER)k2!4xZ z2Qdg&J3v-uw-l)J^Vpq{%|CBIq%4!-*v5oGh}1OAl0u>0K_X#J@@$>Cj$E3MRh2SK&YLvxX2VjN zcErA~LmsHa@iP0s!}tqik0E`4%=J9;8VB1{zHsJL_2_z)j@wu)#tKGIMP+mJR4rL7 zW~sh>GntekYDtY*%y1fIw}XlVz6ltwy`t|GGwayY9+NYfZJ|;MN&mc318CfYTrDR@ zxT2y{rchLxc#|eotT}4vU{S7$npDy?)9`WAqS#yj>B!-#gXt`0cO|_lc~;`ctc;d% zx=v2(74Gs9%=1(kMKh_+DaIx-MQN-yEO9Iy0cdtpru7)3*NGa$G#;_hfrrs@l$t|L zgIb176l~NX0|2KBw<0NhOJWD<4T&w{NH`zTpTgAf+7w<`jhl(X(fI+k?X)81W%169 zsd{7eV;u!MW3fd}m8`{fI`y5f62Q$(H}jjDVq=@KvCSGD%S`Pf*_^gH8=!5$-5paQ zu?MghPD(*+@!}p^kv;VbGa#Ztm`r$JlM(Q;TSTIq3fjkhFj+|RhLCz%8O)S2DGd8R zScpT#5bK**7spgrN)lPDF(E)k)mk#`zjCz!W@8BBSS&QL$u+tP;dO{Azhb#Xv!gM@ zRtd!z?o1)O4wffhcgXyPkpIKp`@qLBt^ea^cXrclvfZR<+N2uowoThb8Yyi>+i23J zZAuVS1$6~cK@kMG_ZFd5K@miyTEtatPy|sy5mbd1K~+!$p+T=3RBisw@BN(fOtLeX zrPt^4eZ79K*Y`W^<~{R1&w0){&-pv&oSBjL+q8jQ)=io{LXH=xvHwKuyQ2o8Hv~1M zCKDVoj+Sp4;i`t`6lwb5BHYST5v8Wm8ix>@$d|KF2sB%6wPQ#Ky~7zPC!}^uM{fE;VF@IJN8X{i*I(g9G=Uysayms}#XO069W=F=GP-hB z332tKd8O`>?nLDVS4>zI;YD)i(G(Jy3WQs6Hhxr#Y&5PAts=T3)k!Q=m0=!wXFEF}ypl<`wtRU`C<=hALj+kmf zVJMkIq2ofEqVuE5K@Crv*52$cnLemXQAI9ElR)+|cSzvzmK zjJ88gsVZM;8r$rRIb+~pOeUe%D=MV4PEe~Aa#NzxVx9w6+T`|kqp+W?Y+$t3mt3s; z^)B^G*ZbT^h!QpZjGgwXM8hT&YfS!}#l0r# z2$@eURgsjMqQr(QqC*3gk-VsHK0#^xlL-YSL~mset@Sh)d^}BkUQvV+k7s=FT!L6o zc&^B#9e&~@oY7u_e#@q^A}WwH1|os-pHox+TyFI*Vu1SR8L0k6 zMyY?^LF%6`TKzNPVPMyv9G1@=^t&|d6Aq1V45W0Me#QqA!l3cJrf_J@UhB2zPY&bz zVd2oo*KnBeCr9a@95esqFuqe24vq3)piwRiG|Gp8MmaIiC@%&Y<;Fmx{1|AIBLj_a z42&IO{%3@1VBApiKO-In4vRPcGvZ}n!f^9HBc28xFv9%Lh_``>2b%vG=`e8QDDyw6 zuhd=`_4GHqo#4-Y`Pb|EAN|OGsJo$WDg4p-65vYw2H3|?cHuFH@Eh_I}w9XJj%EPAYV{It1g$EgRh*79`naRjiE;j zE}=K{qhmT3Z`#KN-Wxar`!@5=oO|LNydIVIqMsz+txL<9bX!I z)C+(Kz!G2pxDuEKtOAw+Hv=nyjlgPP3ozjEJT^y1@#g+M8T4Q!=r zJ*X#*#WgSom>}BqGP(xV(6s~KOQUOGToSIG?RqY-5?DglTDx9L*T7wL?ZUSO$Ke{7 z0ZeeW>r3exxDJ>Wfp4Y(tAXC}$cF*#`ea}eun1V_Y1b=(t-xkrS!BDOJOS=e2oKl_ zTnVfm)UIy_)&ZNz9Vqa@uIgxfmk!tfTmozbt^|5w+VzdV0B}1n8@LCU2NVZEZZP}- zgTQ=XSuDQy2TT~!t~UVFhPLaTgQ0gAIJ{lo0jwT@bRPnLiTFkzu=>Du zy&70I65sa&HjHZ5(~_ZQEbI#mB;osU!0d7Ddg4U5k8jrtfz=ZbPhegE@dG9t1pB1G z9as!3I~d=a0|tRXVD%wzKNNbB+x1dlU?S2-*T5EFLrS}jcgl%YU^Xxa%m;c7g+1vS zSVz~B5Kmw$5U&9gm4~{HE(?5iel23*XF}34IYA`VwGu6uvbG%=2~V+ks7kJM?B?!;lU=IRoy)JM;`- z;Rt-w4_G#;L$3msjPB6)5GHl#A`|Y1bm%RFM|S9wXTjgJ4t+V$GrdEv2POb_00Y2Q zU>dLsm<^1Xjqrga~GFdJ9}%mdZ}3xN&565uXi8L$ml z33SatIKTv89WWW#0L%b30r5I|(F!aC27ya~o})VSa$o{*9WVf_1D4E0e$aI$@&lNc z3wr?zfiZItzY}0DU|v4-0}Fu*h(8PZ>H2Kw2bKWK>G~Y#2bL8;Kd=(G9aw!X^aHaO zcj$Y8CFgbMo_UCGVTYawOuGQzd<2$U)S+hqD=&dw!Ykk}8~&d|KFx>zkC7k1z>lyy z;jggo(QxlUegOmCPQ3|OIIL4oItKa^I`tx8Sz@QY8CW>7Q}^WHdQ7LD2doZs>g#}o z6Fc=bVD_X=J^5JJ>BvrfDKMDTsc#1sW_Rj|$Kg7^Q(posEQLN`c6q1nJs$R6-KiG? z>(+JZyMTF*Lr*UB)Itxiq^?tMAbhn`k2?YJdb?B42L`sm9aspg1Xk|o)LVg`FFW@GcS0m416OV0o%6nE(>fu40; z`YvF}^IdxK$p~jxmtG0X{-#TB0jBNl(o+{A9(tF)6qq2o^#))oFzyt@XHd6Z3Tz$O ztv3SGMtAFz^T3Y-53B}O0Goi-z*gX9U=Y{{^o;M;TYw3`Ag~1JKNa=|P6k#2o6kqQ z4(irZFMz&@-THE1!eQNd9o*}Hdr2;>ThBNR>6_85uLOG1yY)86mCWwe^DacZjzzq| zm*Cr+UBGI5pK~MlR(zvVEP*_}%~^Un^a3k@dB6qWD^Ks%*8v;OK|H{>0w*s*{DIlP zrp4WQ%tg?DKKv2C1aiQ-i@Ws#@P);QC$O@jTc2?Tcwio|ay9G$Yz5|DjQHFO|GDr8n~}cD z;D0x+frY?2xR(JNfvvzGu=01LCm;TR1(!oE*sT}gx&*iu*Oi^HJFcsNmB0qzMqm@L z9@q-p0Sp3Lfu1ge2TbdRea?cOo^Cw}SO!c3)&X;X4ZwV0wcf3l0Sg^H`gUL?unpms z4C~Qbaa{&nSq#0Sdh}*s!ss4-&pAlnm>#|2O86Vsqc19e9Iymf4O|Iq09FH=fJNuQ z{oo#bGq5hXNB0-PZin^gg}}n8J$g3EQ3J3P7&rpg$nQ#EH82R=OV?>VdLr_z0q8|O zrA_bA8*yC-Y@vJwdXaBIU>Dr;j)ERsHvk)uf0Z+P^rg6N1(pL7G9Zt9s>|%r^MOsX zd-NLc*;zgMF7gLlg8XTi+oLDL9tqh!`V6GM@M!2KyXC;&+0X|p1Ew8^=YN2C$3y-K z$mjOx1;E0S@hlMR)v&Nf&jB`_(xcacuRN_sFUEDr={>p&cC226=ZSy~z+zx4upaJ# zGY~H9SOVM(44j4Z!rgOrkM21aa=^e&7#4S%?<1kU(7{9S`|5<*l$2 zFrmChucYfckPcvL1=sg8UKWydytR7!b+3_VBKn@Zz<&NMf!k+YoQ;QeIN1* z*l<7eUX5@cKzf0N4?=G#uGjbIldpl^4Jen_LGL#3xK8`9M-KvMBpEn0dhTgAY#{16Jx?1gyIgb0$RaL4MJ6h|>Fl`f~_J z%<{#|@+D;Y{L5UoX}3GIa~-|}$$3un`7blhWtbZw?s*`np9i%1bDm^~&hhy(jdP?K zqAmF4A|4aSU#2f+ne!|kMV-<|hvOf)IWqMF*1Z3?5F-p5Te!!JrN&c~F^10JjKSCNWt)|58|tczQ=~g@q>_ z+2FT>cauC0;ue8x07vPjgYu#X+!k=?rlc(keV!~IgbQ%F5^kP<1@(AMx%IPisVmgZ z+BB!Vp-Ww7+d0hfmAYiS_9C46bwNFf;(&wV>Oq*B!5L+O@;wn;12}Y{GLKNKsGMrI zIh|+IDGGvz{ANKe26M7xA3BK31D6OcG>?{mO8|#1TplD>3a$WLS_oGGt{mK;5Uv{B zN^rgqZZkNV8y+3PHG*4bk!t~0W#NM0)>=5P3vG;r^Mk7ZXT*i_a56ZG-|-YK%y0&k zOeBWVlMA;ka5M6pxB_tX;JhS{gSZlKo53xXjN(>;OTb+AfgxNKxFm2Bnacszg3AE6 z1!z+rvwWovQ#m=_Wmggo_djR|Kxu!Yv0^1dftJ2l=f8w*=hXl2P17aKRA2$#AO&w-?+p z@@#EO57LZ=5H(n}$@K2*1(?T=C%s7P$y~2JNc$r2V~B?|@>JG$Klozsvh4(aHuGuV zSAwTFKtA<6@JTq$1-}kFhC9|ebCFSJvf)$$xh}|cgv-NxQx-lECJXbazHWi88mt*C zr7)2uM01(bd8ur6DXbQ_$6#&Xhy4rd{%~PYzN8L7`Ku4=Lntg9R3Eay)q|scO_d+# zWiW?G#brL+l3vAn05z_N+f}lzES7&y3;BC0*CFy#0Y6o*1@))d&p)NxrSh+mQ$9oe zG{8^tmZ1KGO*ns(KGomTL&E8TpXxV)`X-Z~i=>3A)5=e1{Z2)?Cchcf0rDW5WKmi} zII>9|xMXlMNDApbn!_deVz}j68^0=WE#QtspG9`; z4OfjlGJP4Qdh(dVuBn3*>E8ogO<22%#5om?|Bprt^F2Xc@bA*{3thmupI=J51M~OJVJR zE`c>SyRazDd)eJcGuhdL!9W+*@y_GA(3@s^J0JK*c3uEo{>HxaQ<#O|62XOrxfEOi zxc`m3ZGbNC2j(z$f%AY14YLiL3*7%kI%l9m4}91cCe@Q%a7o}Uz&ZCvYOK5;^<*Xd z=E3hD)e{P%7IHOzF^o3&ZP|}75-{MIf%V$Il8y!NTLiypRrQLPr+cmpM@R|=jTcGEU%0d8gWspn6`v20t z@c5n>H%pM|j$a@Bt^j5_$;vmaqJLoN&ZDgKJB5(gt+;P=nckp;QtzZga-{3d<= z=V4SsZYkt0<$UX{W9G7Ko98X?TfZM+c+p`deDT-9$cEoV@cU=BDuP_?Ukqa-{08?U zj2)26{qnD+BL)Ne3i$n_bWl5$2Dv?uTSfWtr{gl`QoHs6jflK|SPoqo*yB^cx|Y%H z19Vb-t)lqA?T^|Is;>Agk?sX zkOjA7$TdJNq>cKcHgU0Cn#p#>&?B%%smgw=mE|+v`;7?GU*wandgxk!eND&jmoDcg zc5%w|<;$_K7n5gsEq&u9>SO)j7J&P!V`17Kx8vKsF;XI>A6z3i6jP(@n8v~t?EY%F zG?~Z3bHdn1j)j{k{=4nNqPWmLJaQk&aXEo9oX6$l?*9|_40b<&{U2-bpZ^E$XqrVS zySvCg4$9LCaJh)n4fg4`^u5~5FzJ`~Jk8Km(b{K!sw;cJm4jPG0ru9Fe&d2mt^H>J z;;EQ8X@TF{`}}f0eY?|ngE8Hr=6M%EmiEfgTx$4mw3|ckqtq-f+5>o53;%&1*$$R? zn#PA(f}`I64^|=_&CpSdeR;Q$E&ou*=&(AH5>U^um+u~19Ui zCSgD4RpE4*ZQg0GOWC{&y7JnC`umi=esv?HTsv>JwfPKq%NT+71UlqBSm-z)Q_TzU zlF)e}ZHX~eBJVRw;lBX;Qa4i^LjBv$J2=r0&PN1nW9 zmnSr~qhSv1i(b$j)JL%{npdYVO297ye`UCGY^oF50NbjHNe&yb_uh0$(XLr&cgsv{^ z^T%|sJY@YJ10LL8Onj zGZ8rNi&wwhjUXL?aN42 zxH6Im@2wPH=n8kw?HQ-sx9wlN7yMDY7eH4Lo^|UF4(>tvIUTR9%FNxJfG*QH}CD3r1=^g zvwV&CHR0Fnm=6hUpBroz*etL|GwZycNh$K16IW+IL|*+fR81y(IGH37+pO z!ltx($Hl;2UO?6hS<4EI}5(FA?lPsekY&M^9F%zFtB zaro+7NSX^qbvZlil*@Rg9)j|E2A(&=+No{+S@xcFdhJwY%Drba1}P_0U!EVY~jZTdJ4(cYw?P zq+NeyKlQKw1O5IKtY3WEu8$0{4?SDk{KX%{-+b>JF8b2`rO>s!8P9Q&t#LHtR{^f* zn|A#_)JcVwKhxM$3*wHhce51+r*nL%m-S;x`Z4`wv=jPkx~iYQEBAxByr0Y&W(Hd< zFz!aON)E+(?)T_(23gXm`rLl)-QNe<)d|(!k-i4#Oa8rGpGkRsOvpYlwi)+H4L;B7 z+uwQjJu`bsNF>o#&=?~D@k|W1TlUHIyLUOp>{FG=1@M~zzp{Qo7hxf|8Q{1WNp2~) zG;myWC07n^vV~g*F4e-(^Vi81ecQnWz+t&t9x?da436|+%Bz>7XSOMR#(WLQc_txW zz|lD!go)s?E&5WyEwFG|;PNe89=IhIZV9*&aG_z8f-43m?@Kt`3UH+szt!MYTDZ;N zD#3-?v=Ll|MXm*0m4yp}+X&7W2U488hhe;L;r!s%S-8pINMESmY;bEW+#+xkzfid% za1=jo4yB!!gDbaimEb77p?)`lE3?SegIjLlc7P-Mg@(}zZmEUq0!R4~Di?FO%8yVk z30x86LgSJKj`BBDE(cs8xOr3uaga^(!L4g=*EcXnWBX!o8J+F=7#ai8J%tt@P4n0! zUk=LHwj!bxc`LsO#`<%u|ppk;x`9e!+~bM`QWyIiwuz~23J3_Lthi3uMAuc zo~KU@ky{I{(BGl|ErhE9SBvNE^Fp{S;KbMt{Rqwsx@^Lqora*!jD}}zv4|eE11H$OT zR`-x+%d}%{$7-+z$}Wv98=%1vd>Hl10RgP?S#O~d^8 zXQ)@<;%3$rCIiYH$VRaox{8|dt`e*LP2>F&Lx!3d?PDZOWwaiC<61iOx5Cw*dM&j6 zINR(Dj`~eOlfIbg7@y$19XrC+pK{Z@f%7|iedS~p(w7f?NxygKi>+x>WuD9#8b8f% zTfZ|Qru5Z7Usgwlerp1h_s_$5z8Z6Sm)XZp7IlgrEo?21?bOc>*FIYGUGpdUa-nY> z-cQm@;}X00m71TG-DR(@l!|BpepS#nIkQuz=U{C0RhiPOy=z+s=J={K=NooGWx|ER z@uZ`DI=WMTJzU#lo_}&aX{T%srk}K@Y^MX}!inr%gm88&=+y7@gh@}Mxjf|CmIs>J zR^_1{`jXD*)a&i*w0u-5wEvh9MxW7t#2kh4e@>@X?}*zi`W7H$1mS)9$h^C&R4)ZlmqvkcBx`Q(eZcJe$^?#FW0J&{utZr+$98GH2=g z7Y?-7M|nqOi{4q)u(DI1;Kzks-K%k%;@1>rZn6erU(%PDf%t9g)ScnhV#>|>qQcC{ zmb1Pk&{w#%Q-7Yv)|PP}Ciu+u%e8$FGZSK}-`)&;O?W5Kq2bzA^E}kDFl9BL((1{? z{mi?a`VqJ%w6~pUY@%&D*gh|6FdiX&dC<2JZ-375?Vr9$ws`@QKwkhVs!5-(OP3vs z%3EGrUx8IQe#ZcI5mUX}0oj-#U3$IOHf(D-z1BV@RkFSE&qDuz_c{G|sI4xvFXl4v zc^D%%a~UXtzNF*3^oxdu(N|^aU$q|xhlw59r$VTp_o-zSbm>2g38&9vD)WnM>m>RI zk9O>+FhNoCMRrY|jrz3$@3;!r*0!1YqA+bxn?|}ypli{sU3zP{vTLpjVa882ax6yg zdJ}ha>Ho(3xJ}u$jD@u;!D6XsS>fH}xsiT&CLeAxOs%Y;UrEw8-Wk1rj zzk57Zgws_BUDeRF9lAEymuZZ9%=NPyXa$@`k6$Pa4%`Sa5u5I$iVpG}=Zqsed@nvM$@$gNyZa^4a4PE+)s24V2 zSk^?GVN~VIIjFd~i0?aHz3&5|d8q(6FE}hA$%D8V-~!<2oDRZVa7o}IB%`z59`_HMTpLUl$|y6UEIFBICdGNB1!~@R!%rrQX>vPtL*jhKJde z*J;|FUvtyR|HHK{<}qBH?T{Ix zGfI;mk=ycBm;NBm`_>{Yd#je2*VJ*_Cf|Lq2!0EH>Cz`tx$m_DQhXc1bq6}OYbgd#n=~B;DnOAB;m$R@^Lzd?95C0pX z(?kAoAn9TkxN2}8*xS-N{(e53ElF44Xw2_>yY;8{uS~7BEmP1#WvUpuVq&`Wr>Fw= zmUW{{@z&jEL&_BDKczGg;x+JF3_Fd%Ir&}Y#JeysLWhW)7jK07f+M=sJE!K7dk)-R zm+krI>dB|kXG=1Jgk!Hp$C z+8lGgZ1Vu9EV%o>?bgpBcU5-#8oE=BhE|oyBFGj%_S!yKlyf8R)n3hihGadWIG5fo z6M#lFK?VkXDXyELtM%{Q`T){}gYNAb!L@+9p9mRO=UQWJWe?m6|Iw}9X9i~~*Wy@| z7sRLAR2JogTxupYu^FZ!@wIK=45>J%%7L!x8@tszq%amg1p!lD7l5w;eDEsU4R?t-T(YC< zIP6D&Tm;GCAT9x%cTKnHolVK$Jm7{)DJ7QyE)JY4gj)bE#=;e{zNiqnrK~TMD+lL- z--Eexr?}9*5kKBVIxU2&1DA7OUwcP#+rednO9=_18C(H4BW+}tz2Ne}?I1aGy?D#0 z3mI@qJRWwqzgvIFS~k!a(SWM=IRkB-!@LiNS|e2+^P!{Sfo?T!lI0Qa{(&3CcPZS; zYP);it(6b199%(bw|+#pHrq6RtnCg{9&;Djl?)7`Wcm?C#t+>(-Rhdte}R#F*-j_s zqP>J%Hl<(H>wa}rOT*p?OWmY4J`aAE{D}ASTK&@QdVas1^8~Y3R3lj@E8sW(7raB( zYIkGK)NF&Wdx$Bm&Cpf+E6OqJl6#(V)KdY@PTKLL+5ry+or%A9Tb>0ow@Xeu`4XDc z9AByvFOLYNIB>sWd+>;@S%E?Cmn9Lhea6tYP#;l0B4X~pWW`M$;(Cc_(1 zUZ|~ZguWKMFE^9IMtLhXT56nA``89|*Q?#V`=Dk22tEcsFB&Kh+(zM?4wCnSiwh$^ z18xa$8z`lf9dp6eBK*hg#|hT<^g7!y2g(Sw&1<16h#-dZP+7`@dvZdL{xZb_2gPj( zxTfJf;rG3n>-Cl}MP1HQ)Iry-1AFvC?bB(U^N9Q-T`qKx`7fj2rB1BZ-o0Z0tFEcG z!@^m<+_FE4ZucPhSZ|Kj$uNGmHZ|T+x?e7&$1C|c0-}2&Y9|{@e zM;G+ez16KBWuIE1;_$@6>*G|8za_ zN4lD!D=Vo-|7W=TFwYS$v~7>@f{0YuEBO@Ehw(l7d)n2)*xv8 zjclKlhwlH>ITxyvxKmm41>>$M`6@TR)e|&u*39RzDhE z-C=m5aPr|N8{f*1;k<6|Co~*bOwe~P_HOUikEZlKfQc44R?{+^YT84VhdRilrgZC% z!Gr6cO0Q*|0&_X$b);8^+n44xv~N_8y^C=FeO$MG2G%-V7b=_d(^-C7l&Ntzb%?!E z$YnrhK6D1(?Uw5wuH6`UhK?0XJCAN zUbmdPcWqPlTZR{@q92TLB5;x^f=nx9FbyYMFB&qKu3{MhWfhRwTiC517Xg{a!sJ8E zji!8=ZFaUE~3dOA&zMQ^aZV@3Ei?Q$&7>xB_RO{{hHJ5tpQ3 z7Oe5cr~)5U&rT6%$p7$jQHm%|!3Kf48>7%S-WY|agKmskloE5QnSqxRQv@QAn<9=$ z5$C6f0u?9QCt0w9ePh(oDdH^YYk~ZWOGFOk1;5I_;eG<&Zh4t(@Zo54x#`_2XSM-6 zBL)Z+A3VD@MO>UBPDl~gq=-dGU|Da9&q)#INq6VpQp5$6Xr!sYS7js`{t=zaQpA#! zn3H_flxRd150Bj#b#h8fo|!?Sk=)QkLn&p%>2_rWO`1$JY_r(-4+AVr5k}HQGSXgg zCen=W)u4z9mxi=sG=(u0EgIlyJWM9aI)oB~3NjfoTOeQYA->~+vg7)3tYItM8qq`p z#$|XsX`wMJll3A)`s;$fmCL*3GpVj8{YJYg{qg#QbH3pZ%_#*)?Zo7KtgpkLpZfb7 z;V-mK8+lW2?)U0!2XC-F;!NZ4GU%+j74uPX(0M#`_TKBFJ|bW_farZ0s1`NYJlP~; zNBtU&Eq6fI4%p*3s@uQY)*s7Sp!Qpst}>s}n{*cHn|X#u9>~*mHC*f^1r~y zM%64TJ9WX|M);dW{r{+5f9O|3+CbBoQ#;nS`$rp0cF#E*CGi@qfgrY*JZRiY>RZ8k@sb3r;N|8_v<0#}c|kkWMw;$>G( zi%g?9y!g&mrL2cZi%}*9_UH$ZzIa=G)_NEfMqdWSuNeCF8u~uAtw;NF{Up`VCvZ13CbB?!=jV=$>VJ=JBM3e-y%(7ji z#YgmQWRo)ZZ^OJGy`#`|ysb^FWqW=Y8_Tx13Hnk;^ynW$w`+v0K8uZ=74}6SL$1vS z3UU7leX_3nJ~F(G!_*b}wp)23+xVMM|NCcSve8E9Sc~~iSq?vhj{fD^k|ymB%V8Jv zEggyP)RE2avejp8&%?d9JD=J)`cBzK%>R8!Hb1?88qMvT<(*fuD5!R>0(!Fi9Iu)G z^LVvDU-4*;*MPqkFKVljFF<>VIZ4_6{&=9B{jFs*Z2Jp!a@@D{Ld@Oa+jrLz5vJbm z?^<{RbS(+s`+lSgM+|HOSWP03$i3DSSYf9Ed% zfBEpY9Nz`}Y=8={Z;yob-T-4X4<{<)Do7bGv=YRD0S2eFbP^WIO+mtv+ize^QuolTYcJdj_%BD9$9GrJ^vHwKF&SJcIJXzH~;Ruxu`j&C!Yd!ijGLoH5EOQMb!qihrM+tPTeA}FkmEg+2S<^xC)o@!2 zw^Aw#IEdQ}&Vz6A&TvUea*g07<9obDdCh=R3%FEpxm2%v+iRn}MWybYfY7Eo`?a{p zJ9rCRiFJX}0# zprhqWeE<4SE}`j-bwKdohk z=~pu-FUz1S8Q(KL4d--B#&0dS5^(nt(Q9){--Ne^hTc*KAi4won&I!MK7SBVn-rWI zj13B^4AIAEi@)zHFBBL0Hg*v>D&}<1*gODk3Ahm$FWIEE-}4FYg?Y%P6l3OGs3?KH z8GCtLjI{NQ&5tvN=7_U)o5Q%mM&;0Vb>J0kyc7UsI4EDYLub>z9{pS*u=o3R+%Y8F zrorc}t~tKIN;g&!Rzzg`(v}UF=gYWQb|BW0D%1aW`2W4nCZtZ+9hV5yqbrVn%PU@S z4EiZb>~I{mCrUi+obq9m*zNScA0^gkp0}e!h0F7Ll-TLY`Y1~L%k6n3N<8Y0dI|Dw z_lsZ#xbY3P4Fg_;A-?up{dJVs8ktxdCDulHe~A*+QQlXhL~YdLaHTK@i{&U>%bk&U zF61`n&s1O~AEPR6zi}F0}72ib77Gk~E3*$cRJyglg74IP2cQp5&Sn+|@ zjkvxS;r=;R?1=-X- zgJh%gh=(G?BW^c(q9@!pLkrn>ilE8ga%W`wK;&ceD+9$km;1qi;t|(MMT&P^n-Jo%2zN)MD2q4=ri_Tg>1zYs|A`bEBE7*#@l2!_A3J;{ z5`#kWzhW% zkKCQE-7`A&`*f{ybo8z1+RbC&`0p`s-$2#a$gk41`mwQJrfchxqHjspo=Jk*`^Lq6 zmac6X7x_uL_V01AAE#^YkB`17U27hX8;SQO#C?>m?VAw!Ub?nE5c@t99u$3Jy0-ct z%&zJO#ccz9aO7L*+RlSx-%QtDI3)UpbnUG};P_;6-0Pr|BVSF|R!oe21<6YBte>IX zk>ajO*H)!;A-C>4{DjY^Yo(Kyg72DqiVz=8@!ors_T?0>p052k#rtcz_WKm?kLlX7 zsornXwcDq9KTp@zO!a=4u5Fm=eFq7c>U|?!du6H@Dz{A?ghp=au#abGKTi!*&(KyM z5sLzL#IVn%Yg>*8d@^0zb3_`3NNK~~o36c-7I$ozW;H#k?!)K(80)HzeDj355ZOzo+7RI8c<_v1%tRU@KS&(wYyF$*G%iIGoC z*M1%4DW9&D`~8nh*J}JrF-{#lsCuUM{piFRl;|;sV^L_#AV@Th@vNDtHI2DpFpAva z-fxc5zCRqs+jscz_37H3lLKWlv{xp35%+DAy@+<}WFNBfp(!!zrfbhl83JRyJ0<4c z>Dt#*hTJ_}`)vxete84-?=)meVA(Y7y{U7Bxci8azf99M9TB*0n)byJLxuSDh>;&o z(<;&epB|}gPKy>|TUvDeH0{&0VJ}Y8wjLS%(UICmNA9FCHO#&Q4IONHuO_HhEOi{P z<&5k+RD9@6p{8NnuZN0ySJc~+M1y9Ie#;;BluMTt9r;26q@i(W6d*g9?ace^22dUzb zgqSx|#g_@uzov>+!{cvH6^{?cxctEb65mP{FC7r`Myl9*K=ig$@vjkMKA0pv8F5rQ z!b>cC`7p8kz)g!IEoQpM^+x6l|`Uo+U@@wlTMgR+oiC|YQ}CFgK^91)6J zN1W~$Cz3IS^K9~pYXc8>#Rdn=Uf~>5?iC%*FWpu7i*?Q}*o4a$MnvZhUpqm3<%oOiAe1ZbbrZx#&eKuHwBf&u7x%g1mK`KY z-QFk0i-iedEQ;13*`JhTk?`TsJW8%YbMZ=v_Hpv<(Ek zHF9|OIPpwW+&clW%IobMC;sIH9rO;z8^^yL6!&64H2Azfj1xP2pdXDM{@FNjPfXk+ z0nriT{bZb2F&Ol&!NcDkC*F#UyC)zv4)HdO6LmvC-xfFg>2abnF7Aea_+qHHW}Nt8 zDCqiO!|xp@9*>X1vv}nR-n++%wF#hqNf@pti7$u8eKtY7c7V4dNxXXi=yfB8w z663Z`5PK87UnYst13~XRaQItE;`Nbn&rT5QM|rm-iKj<_F7pq6B1!z>k6Q;LjrKm0 zB(59lh1qT$>xGq8kM+VR50CZ222YLkBFQg}^&*9DjrAfyAC2`Q{$GvtBD((`>qYF^ z$9fT&82R?6CwD`j3{V`hn z;PdW^Mm_gJ%k|M-c)cx}3WyAEFvcf<(#*vCVTQQI9DFJsvWalFeR9tQcaV-QNs^Nv9uM2qj8?vJCz zPtJ#s&O6*+I&sl9U>Ssd^|WX`DUn@a=Fr>Kz!>ir?MtF1OI#?v)Xa<@CyTxT^!{7$Ime?OBI=l@oUa z7#H2)q|;UM9X+zljfi|)bBrs8QTzO@Imj`J?v5u9{?g%?LnUChBj$GpN{jnOr-Llk zE3B`X2%jR#dA*g7N=t~5qvq2Kh9aW;UDs} z!*L4YjM%>E5N|u&A3GfHS>pV^rlWs3)E!fIJN!?2#Es6xZw88$+OWrQ)2zkcI8ao% zPbrN=*{?#x@A1%m@IxMVwMSHYZURH)rB!o$svS1yy}6=NOTBZRSQZ!iYnFH}ZrJl# zV*Sv#rnzF@u(%y_#m({b4)RUI6Cm^M@aIqwZa8oR&VM{G_MN$6-N<1t&J}l!8v6WP zQ9UYt1AL7dde>ZWi$A_+j%f4`ZJ#5)^~c{mS3EI#=x1}p>!ahF=ZKYKhQ2gMJUAxy zwK-zAu|x055+9F^{dKl@B5CN2v&HL4k$AOCTT<*#S>pO}(Qjpm zm&V0DGe^8PF8Y=^;^y(OFU%He$49?ATYNk|cI9mG{rEV%re(u~*zdE%^An=q%o2Mi z#MaFbD+19s%@MB#Vjs#99|oe|nk{ZSDE84TQFTz<16ksWgJN5$Ir_dVQ86)g`5du+VqABY_;Mmi z-jWhqnI-N^iMu09e3G)<0qV8GW1q|tI}RWEV3v4sa%?b5yfZoOWkhT8$Xn-#=cYuj z%o1Ct#D6$j+%&b?5j;}+GOYmKUOy6Rik}}j48*TT4#zz0H1{2|#aABjo*%WBgAJ%pA>= zT;%x)*Ez&>R9xk?=Y)jw#re2jknP@RD))!HV=edl-|kVp&JZGF*dq?d!gBt8EcpiXPeZYd%Jfj z(ui9>I%+Uk6^f<5C^Dt_lLQg{*L zWsKJNUd8-U#_JeYFy6vg&UiQDy^Iequ48`k7pi#GGGE8InekP|HyGby ze24LU#*Y|3W&DEiYsMcL+Zcam>|u;NL&YP3F^=&-#zPnnXPm(}i*YXF(TvA4p2E0@ zv4HVn#;X`_V7!^Jl5qoL4dcs1n@qsUk3h+-VZIGQn;aT?=X##0&3WxSj*lW{5IGR9jOS2J#4e1*}P&(`avCFY)@ zsEKCvFyGE-$onr>=`pSiOt{GGzlQxaTKv~o{Hxxo?`Gb(-aANXHm+y;%-8vr>w3P% zDy2Gd6|fpnIYq@Aw?XoVIakFOyWQlGz=%60c?8%!k*|%^{nn#jLR2evpUO(uzk=yF zI#h2Ygdg{~bg0>Hyq*ioE_Beg13HZGu-0Na*c&!3s1S`_8nf^6z@(Ktj9iH4>q1O_ z&|&cOT}wKQc&BneMn7+a->Iwg7@|FVU6~}kxH-{qyt|(l=j%7F8{Cp zn`)^ChC!xsd7R6*ka02Nm5eJGS23<<+{Cz*aVO(pK(0nRK~fC3mF$PUdgzEaTViw#!ZY{8Fw;%$GDF%0_$Nie8%yNQyJ$n zE@WKHcqQWs##M~#88(4kD;ZZXu3}uzxQTHq<4(r!822$oVE$c(&p4iO zD&t(ng^Y_CuVh@oxQcN-<0i(fj5`^>W8B9W!S}br7{@bCWt_{nka02Nm5eJGRVr{_ zS8TN424lu~vu9892Tnfwii?Y{@E@5nIVE-Cq$}jrq-zdKO-Y@cGU*`2i*uHob@6%r zE6=^`^7Ajb*gt7%3Y^W#ELR$)*lEj^_0WV9i@a7=2o3*?elgTl6RzI=(9FAg`%N<+ zA&maT%nuMozhve;@;lC<9Gz=opfLKyP&bWwWo|Fce3bCp@?KGH$rq)6Q185kS&y%G zykX{X_ncw$(0wi%cD#C?vEv7ek~i%5SW$IU|8k*;A;K8f^l2wM#0g_OW9Em7R37J; z`C-DYAJas<$i2ia+=SloidoNak!@3cHF1C_vgM7@Sf3Ue|GC7^dLN_C;wALA!$Hi) z%{0f8hDT1(mtPZ<8j%llAU>CJe2nqtkvRFIaBGe;>z9Kd5i9ms`0Hdi;B(FLcQM~( z;U8wc$->t$?>WJ&XB+c=3;zZ4ITrqB=1VNROC~&4Y_#z4%r{y1gTa$MjrlyQJ!9UD zlOaj(ILpF`ojs3dJsFnyz!R7+vhW4WTjvk|0iN>FC|9_Y$4!#&U;b(FlHza7PqH`` z;7L#adPLvBvWx%MQvLu@v)#xSxo#pRUqgEO>Q|84%QluT0&kF{9bv!;er+ExTcqnk=JBhI%rQhSQEddmpAU%l>DR2*K>T-&IN(Z7D|tJ zN5#|NX*(?CV=8U_!?BDL@GbL#4Ra)}>A;Gzl+Tx&(Xl4U33KAci}d8KRdQV{AH#g< z1ByS7<>~zqBp>sj;tl=D%ojammY>Fa+sleK!abV#=6c2RG_=TLKL2&a-@xJ0_qSv` zn-za0^YmS2;_H7?JZ-n2<2L3~JJj__%vUp?(WQ8Lu8xkEnXl!czRi2n~RFiL?vi^G$9`OL-JK)l)U^tC4x$3-k7h))?s;MGoR0fp>)zg z@AD%)#{9YQ-RM&0i+H|#Bk#uey-kK5UU%5Q z{P%`DuRn}IS5L=}%p2&!kdahyKSf_|&`Ma4n z)+^Sq{(G1=)-O_7{t@O&c^%_#%-1t-tYhRc{{i#Hx`y!$)t{KJ3aIiXztxGb2VkOx z>|Aq!;^lWW!TXsnx>@n^`_DE38ZBeif8jS#mf)gfq$R*^7j-kztaf*2Sd;Miua?O(V-!} zq^JBN#mjFef*;L%&1Z^#h2^I+U(~F4`AtK}pTc}@i{jARPypWfZ*RTlZ!fM|yH+D_(wI5AugF z-(dMe%p4?obpkpP#Szdf1VUtC{zor}#73K5LjSy-@LcS-3SI%yx~<+s}4md7U5BHCMxLw@55xP|%LRf?D2s{-G`e8#AgQ$I?_Z7iQ>k$;f+N(*1hdYXBDe*^1D!Gt^M zum2JaFAn)`XW&E3mw&E!`Ce)8BV>7r6)hJ1Dd0&@>bFW>zS|k{ud#eg23I(6bZnD< zW5ps1{}JoSAEfk5;r^ocKFA_}IocHpxBM+7Pw$JP!?-`HwaCZF;VvH6vhZV>Z?o`I znNP|z$A3O}nJ>qxaOwS|bmW7l@pBcAds7uHu4CTA_gRLvb<8(eY@-@t_v+%DoUu%iyPVi)Z{|*%&V;mBV+ZMW?s^Rsq0LT9!=G!d% z=|hyB+>ewVx=*0vYw%>ZT1$L>V?LN=POnGa=Ac|z;yHr(T^2rt`Jjc*WWI8)S^om& zH(L1fn6J0+|6qQPg};^ggn4HDRm`UzZRVc=Pj+s*SJ_8?BLbKyr0aD<8R=}{&^NXmolGYiRW_WgW2Zz-@|;N#mIhOfSOFWMNPxd#~ca6L9?aUkNy`xAij>VX0 zlkF0>^M?GW1jQTcyBG$@<8$z2=QvAzer7&k;oWjUHdfSI_~FdAS@?;}$K;sPo5_5@ z!k^6i0tW6kmZiuoB9{x|0HEc`&Z zP#r6ZE&PGZS6TQ;%x|~wbC?&FdCOD5)3~CbSdA;VI}*h#A7ha(V?JQv*D}Ar!q+gr z#KLc3zS6=sfv0xZ*ry7xl!x%=&qgCo3(r>+>oq6l}f_EgvAJ#20m*+9y zV_~=QPNkQp)dal?&uBw9NC_u~;m>6~RX!EZ2^`EV%+~~!mhG&pius}mEbmtiFEXFO z`wE*?_K1%q&z?iz!hHL9{s#FmNLN1Ft$_W^M1x7`t)U4b9EUK!2t3Ibe`^dqS1U?c0}mX6r}(7CsPdf2 zKE5&ZH!32A^|vuEE>`^MEZ@a?8l#mS*-gXkSiiD^f2$JY?Wy84@Dy(2?T1!RnIBGOIiLS$@4`BB#lw=DPqI( zZH<^J`4B-qeUjyyUsi-6{{r)=s}z4W%Wq>o;|fJQs>~-ok~|BAKnHlTTSmGPq<53j zQ7~4;v$;@RGc7J>zK!`zulYg|94&E|LUSoMb+s&{uy{nGOxxYu~;^WqIfEMq<*iCVGYz3Vkb(DdLQ=@;>;oP6L^E(bB+xL#dfz#a~hyy3eK zFJ}32j;9fyWz5HLJ8&?2SrtalGhz6MaVnmv7CR&`U;dPee+uiL%6vY(=mZCDyW}yC z`I<}A^jZ^YpvdBX4lwU<$@02UXjgs%L2TvGF`O?#rB$X#Rz6v8h_%zgI zloRVZ@;=s6&h|9g3eQ1Ge@3EP>g8!KF@gDd-ZyT%e`+@KrH3g4{3d+~ai-+?A_T4t z!{5z%YR*-{a=4Du4d4f39A&KgyI9vImM{NR5pB$GmGXQM0z*+y$46!Ym5EMGcb>HnDd0(4{)ZYmRg)_ z+ufC^*VLnDI%rYCHwzg zx9)RZiJI(Mzu(O5`@i2g_dLGyo$ve3ch3ER(jSeC)1l|edk;TLChE89tA>G7z1kH4 zepB<$D}Cno1ZOo8f8L<@+T(<9R`Z!>TyFxGRijsb#a4ax8pYdR6M)rn{P~N9-z)8m z>hSN^6(9b-+{;t=Ar;?vwcs|te+D>2eeCxQrjNnS8P`httiD^GeL_i|sQ6k$E?uVh z(ccw9s4@I|4shm^CrZD6M)7H--*=_7n>@u?4~#2O;BvRpZ@gSU(+A#b`IMvu-ZL!E ze@F3SZTDnI;?;%k2;xUH*RqqyxS z`l9CZBH+{xM_wZ}wfSz{=#}udmHyL;JCINGD{BLNhH=4y|9>w9Kks=|{`l_-#=+;W zk#@7@c%kC9@8wEu=v#nO`S+bE1-W0d{28S`s`JqwEB;HCzxrV=Rs3GXH*|h6dG%4o zK1oy2F~LmBl+K~eDzAj*Iy_2H#D!` zPs#t=nt%JJgz#^)L4S}!KbMp8Prp^@t^6Kv=FiuL-}@fL_uVMC&EpS_QaES~Y5DfwTzP1?cs zc^UtFMDt%$dHcstkr)3_NXdc9ojh>It+zaA7~ zlk<7T0~zR_GfMBAD+RQE@f2S_B7`2^2C2@Dfov{ z@K2@SUj%+R#@!x`H?w*6A5!SAe7>}gbG68GqknP*wa3jO$I{^6VERxAPq1&>!9= z#MT#H3jM26@c#y!`q#-x|KgYe{@vHdd(Vu=1Aj-~>%3O#OW%p3-+4cpT+T~?Q~s-O z6hiYWKA?D8=c#8(HqO^l@|k?0sM6bhUOp!p z&;M5OqdM;oX_|jheC?;D{N@+B=0)<}w$j_!8B=^n1@qH%KH8@EYD7=C8924i@ZU*$ z-mUH5Q~LD?{ZA{tFQO;^vf@W0{QP0fe?8*w+EDt9bLGACtyB7)^J1x&?VB#C{JB)| z_AV*lHk~jsz^VMx6tILJ)J6X7Qu^T!$o(CPyGq}V(61={n$EvAzTRl~ha>}=kM7od zHnd(4`~16LI++hIV;p+v+l0=>`!&F+{A=5!f9*TLzoq!$D+Iqvk~t3nx3Q)WUoPC7 zjPGC^`ia^vmiAV~GioR9)y!H(zx518G#u8s6F8M~=-E=9%as0^Tcp1n^<$Wv9|KPG z&bOprZq$CKm;9Jy6zA0$iJs6-qrxy%a#5b#+4{=d7aU>1^oEga(zn* z{Rfr)@Sh5uiWTPwa4P@mlOD$po}=%*_T89*Cl}{x#W#LJfbZ*j^NcG|;BuYO zHwAo>KD$ZrwJXkLgyN*>w`ll%a&P1Gpy3|~<98}PtQ}=?2FL#-j<0tp{o#*FKD)KP z_Zt390e)B>JD*Ve$iE2idBwk^_%wZZ7{BLhKHmgR?K$)eY0s;a{_!_Rd$#8U&q^}q zS&C==SqK$&#uZj3@VtivNP5L_%AEI_CW!RKi>8~ei&A}(&WZd6t{C$_&Wwj^Bl#U$ok7PaO#(hYhS zt`d5iA0NL%%5zNh3e)SZRowXlDY)%Bc%kBJ>w+7fT&MWqn&38$W`J`!)n9FT>>HH6 zt@hW4FOhWbP<%-3V%m>Hzu!?jbHCjGx?G$G6rWapI8X6EWn6Cpmkp)g_p1V0znrmC z+QH7r;cv`<@w>-x?O&67w<$gy$^R8b z|L5}FFU!Sw4RCJfhomGoXuJKg(r+|`aFf!%Pw}<)3t;W=G2rCSvHey`?)*=tx4Xgh z>Rrj>dJp5!d-XgZ{-y?!wG98H3mA*OK*_&`J6sClzn2AL$~^ht35TCDVTskBU|F!`q`r)n8 zUtMkI^Y#e+&=ZB<(4WVF(|b35QF4BszPFu{ z&wtT;rave7d_~{!2H;8j^Ny5!zN7iHza%ewP#fZLw@7_Au9Eh*arZRES4V|@Ro{E9 z;b#gZ?VF|FPT*9pb)6q`0z0o#`n8`I;8w+dUh#cR0VWlHm*$gEKLdRmntq#Lr%>EW zG@jv7;Ou`>h0%EOS&FycEa_L|;*2X^`aJ>E40T@2xDo{}J*A(1l@x@(O^WB=g) zy!6v`AAu!lcv2oa-_P3fk{>5L{%ZiIcAK7({7r5gc&X&G{tan=YqvWXkLUA7r7yih zo>_h0kwSmJ(yzWk=-;GS{!t43-zdHFb|JKO`@0nSb5UUChZjh_UMtmb+KO-JIHhm5 z)9*gu)Q`^B<(~GR)9?2gm%oGm|78mPkmh6OU0$vH`P7$5{>Sv3M3Xbu0H<=U>HG*W z&c7Xsua8Oj>ATDHdzmZrrIO&bo^{*4+_IOTuj=fd<)oR{(p-$%LP8$xn%E>nC+ z`STLRpToG`1THr#{n0NAXm-bv;_H7X_!l)zU-7jisn9*p|LFJXl>FbGg5MAP3P~FJ z{|ihHIee+e6aG#cD88-u@Hd6r?61c`;iP__j?BMLV?2~K-#lCChyGHYsaSNzfb%$d zy|nWt?Jut?`Ir7o==I{P0w?-?&lh_0*WIe)clFysf5Dk@-LJTv-^t&W#G^Y5e}&N5 zc)trc$%m+(|2Cy})PHU={9eW-kKq4b1y1G6+!Ze8#zHc`JrxRGlAO6Z1z%+xe#takA^oXT;&p!ks=mk#;3;@?qx__qWw z{y%$3>U-q7lEX#X{;R-=PmaDuaG3x6d%NPpdQKghi+>-`eA;i8ay~)x*#`TW@>x}Z z_dcb+N%84-3SjbcLGgW#l>e`k{tm|VCU7D9obo>wSvR>y@r?-nVa2C4(j%|s{A^18 zUrE9LLGu|_eZ$)G{M)p?|69ucEolzt8H(Hauly}U)MHHX!(Wq1nSHcH@ilE=(|_g^ zpKb~LZf%RY;v0V{fcZC87+0de<$IRTy9Cs3aL(Q@1rBdPxfX|46}~g>jWsp64@; z@qUH$mzo()F@@e!dOI(ezZHvWt{DE;Qs;e=o%1TikNv9PXmyle(e$Dx-sW^SZnSR&G#rcTh$37m;=MOD^TyHU(w?+c1gpCR~D6kpZ+OFDmDu35fL>5q*Iy^Y7W^(6mx#DDUqz^R>&XdD{8FUr5~ zFs_*fE@$_JzI2WhM8%48IpYCB{WGWZN3ufyA;q^^K6l9rKc)@6CncZvD*a)d->u*2 z+s9P?wFSxl_cZ@cC_Z#RC~s8-`mANG@8yE)@|*Jv#houn2FB-k#WTkQzfALg5#xFj zxV$L^|4<74NzH%gnNt3uT%12sd`-uZ$&GI+zW$)R)Y{?9gHo@fpB22S@_7?*;;WMG z8!~->zA4Q8enjXml8f`JjO$I{@?FhmNDXoGS3Cm(gz_mx;y@-0|4pgTdvspDQE?|P z_&uBCB`*a|<=l6|A6^&0+UKK+w>5tIK^@qCVEMmV%4vGh)wgT=e_rzWjOL#OPW2i-OX#;~hqakr zi2|3OOTpJP|Md?_1|QY-xfl3_f%Nw0XMxjuk7(QoePWA#UwXOryB1J~vGaAtg(mbr z!#^wl*qDDa!0El~Gs2(7C;KrF2wziwg~`1Kft#$*EckNns+6a# z6HZYJbP3}@O8NT*z?16rGNuRL-YIlXlZ$ge@niQ3@EXNmt@u%0Ul>;WO~8}#|FY8W zi_F*GQhYih$Ie1Ih@V%#Cl&gz_V3dbZ%62VgmEoI@UpnrU%0kjFE`59c597x-5aU2 z+YXMOb-hCsuibNd<@vgoi6L3Xy?)2;Et_|^Znr=07G287<&4}wM(*C(-i}Gvt+i^+ zS}&H%Xbc&PA>%P5n~y=cY#hqPq4@i933*M#6HZ2}TWK{KttK~CrPJ&7`U?vq6{qTT zyv16#=XKm(!>!a?O|R>?Znfnu)?4%Cx?AnFI$gKiKja_-w06&{j!b68(V9s_?gCoP zEq6NQ71wL_IxEgXr`+(|YQND~L6X3oOYIvXU0iI`Q6%m^cVRT=E@a1@MYr24_dMtN z>v!&O=WZz#+}YyZg4}M|v2}akmY?q0yi=d zxURTs^A4&AzrN`mbbDU2+v>Q)(gK#N)$m<5cw4Ci?uelW2^)P>$j_!)TMpcHQ8zDm z@AU$AmG(;DuG4AP}Bn71hm43I^YK(H~q$k0$59o7&w+&FqMe<&* zb-^X=atGYh6=<AV)o%CPPV1n1An#_} zj<*1YZ&tk6vux_KvE!s0bJNpJhSFVs{Y}>e_?NqV*{fg>^#<+b17M%37c0ItiKgNT z9l%=>NV750tP9d?j1-lNkWzIMb46Waq$#;lp+YsJ3?H{r%EJ>x3oCHHE`2j{(@&gRCXl*l-{N%s7iM~Ca8tvVhu=c_d2a)nr>^M zhip=M*`6M!gftT*XU%g&dM}sjwfS0dN_+*Nkur`et>&_uP3a^CM0acWtGG9pQL6<5 z2+ACdUkvWT=#_4!P$(?Vxw8}QT*jTty7K0{%WPMJHcu~4dhf}+Hai)5ZLigK_lI99 z?|r1Jd*b(MWut77g$=ADnET^j6w6 zXxhQ%*Y19|j9YxtO?_*q8Y`HyR!b{u_zvI%0AR zuKbBg=R)UxgohdpzT*~dTha*3O~nS~Xxw5RCDz!H9p72l>CO~h;?8cJ**v%PhD;4` zB3_u-4==&(WE}6k{^kUd>~fIggan*N&R$=*dGn5F`t*l8#?jbNaJGy=N@TN;jJL<@ z9B+nvhU*;8H?bMA%d!0O<7pMo)CLugvNkp|$Dnw@#z;c$$c+gfO5MyH?k<(vo>nwC z0&W9$-p0H+ck?ccRxty!S$CGa1#DIpcg?z!Zn3!x=?bae8|(EUE{iwfRa5dRQYAb{ zV>7**5CF|`eP(-B=aPD>T-B#D*(^JZ=HWN-I<*=P?J2fTFxb7)QfYz3#KSF90~1~y z9tX8tF=c3(P)mp^?(uAHzTB-$2gB3ASJ9#b1x*k64k$ z6RManw`3D0m+bf$UOxwg59qxtcruw)}T2_kPhuyI7cOl=pk? zf&GWz^e6RHyhPVxt*lkSQ@)b!-Gl~vcNK792yBau1(H zjTRBZFq`Q&YX|zC+wOIW<`j{4(CfSoNb*&0q1>sUB*-<$2@~FqSOEz$Bw1IEudd((QKQU8b z%)oiOSZjL4Id>0yVTC<#{oy~jF!#)%u6yQ?cF)`x{WnhkP0)XEXYQGsqW{od^gpM` zlHXIgMBI|S7TK9-o#6a5E_QYyG;Bt4x*&`q!B?qFA_Ic+qNUh_wce6+o3anq(z)q# zQ`WIa+9k6bt}l!b1R>C9n=heZ{Jv))1MhR!L!ZMb8!CrnObag!7((Q=;TKE*sZ%Q@ zG`rZ#fwo7pO52myt^mj84A?fs{+`Xdvt#6LEkxTx7aaquO3c>cZ;)b=Na5n4{coTp z2xj%t_T>g-($+nDHg74q*^yC}Qz$|%Tdj2~9gJ-kf585q?~}MF)(Y7Sk6M?!5nFS` z6S+7S2BUBRwO=3?X8}SKE(0l8uLkEm$H0h_gKitTG7w%e!jJ}6f^(=t;Fcg^2lvAS z;wi9h1al&>cl@{tV-$W(rQLUz$_O}Fw$&;Hmb(VfdV=ebw>&JTgnTD?U;wSpNCM#6~8I6lR06bF|)n#AFaZzlFOo5RCKGRLaJb)(I{~@3D>aX zrVf|3)W$ni78=K-4Y^)u#D@k(TXvjXwIqk2P?2f`RR+UoCYKASXCj5WSQu}ICcOEU zY8t0nC}N9p8xpTdFAJ-bG-%zuD44HZk&@N*YNb+TCa82liR$!P{fLgr^|A3gk45g| zi9UBZ7YnKh>Y6M(!ckN_D_44lgq0_2tu9Zlq>hDT50wszYOABDs?eO{LK|8jw1tH> zqL*Z_(&7tcKd zu{ItMElgfSjj&7sc}Frw$A(GAU;~}3$G=3&6bNV(<~D&4w@mf{U;#L+>lAK64?Fv{x?H;Y)A z+r99(!bqrl5nGVo8j9Ld4GA%)8p}pig*iq|q3gEG==k|TTgc#VVexB;f+tU_n_{xT z?;jfRq2?YlC%i%2y1ZsYF)ikqaq<=vnwdfHQ0Z~qZogKFf{as1^JLD5|ISa z1ZI|+W=H#oFQdrks*9#NT@{8D2USQS3JMe&=qUjca}eo5L0Wzt8s!>PDrPr-Y)TM2 zdjJO7zXLrsG9$EQ#nZhFaq3~Fktq%nQJsQ$I%N@p2_A^?um>VV_T@5F1cj6G*aJta zVClj$An&0rFlHB|TV6&`3hZrELL48$vIx#>x4N}M#2VoB#YnNE@vf6ltNm$@d4;nH zj3r?u)6Nf;yJYM&bg~)HWo4L=-{H|KiDB3!c%I$fVA* z*FwrV?l2N;Hmi||Svv#=$C%8XP|tPtgp6B+SAb`hsI?=VOcITSCB6StCluDpz>bSe zig?MV%qkpiuMPMJRCBjA)+1>wMP_$uUc~W)7(3wm3Ux-@a@v|^Il(cSW;yYrWXlQJ zo!D|Zu8n{kLVD2Vkd08o!=0F;VUksP6p}SFTbr4Sbmo);;R0q!Vtqs-Qn(zZZa7h! z6|gq^OKqC2DNGP*d%51L!OK{lN03}n1{PV!lfTG1#NEMUU9MGR*G#ts6SG-`ueZ@} zVrV4g5?iImktV!{!G&7~+F3Y_$h=@cGewijtf}#g>b{wvnsk<76Uk#=)`9(L=LURb zr#sWBv5qK?uy#rO2i;K=Sl{IuW;-|>O*+7R$+XG#dOFk5lVqMd36^6slf}-YDz@-? zm@=YsMF*RFM9$a)_rNGmKhUJh^Ie_lny%NF4@@t%8AgQ0K(oUOntk&V(XId(eq)F^ z2a_Po7S6!7*FBCqG1VhfKt}KYBAio;ZkBNi;lnH;uHBcl-t>A=18uVVu?PGt6-w6akgC>vYUHWYPL z+7jc&gIEdKM+iTHei7jX+ECXrSB?4%u-CA%Kg=@QeryCAv_=pCedrJba;MuOqo}vy zF3a-#Kx5D#C)OCk3_X%n@2J{@7>wBb((QM2y^b85q46y0F&in|RV`8%#Em7hOe;Rg zi0~)AgtLcc6fd9+?_Y-5SXsj242N}BS}>f2qjZO}%qxMJ>^Kcn!sS!Vr=uLTugI0(7 z4HIqOE1G86rTv36i4Mj&JSjAZ(9$1;SHsW3PH&3BE)+)Ft+sDxRA_GkIrg#eL9YKw zs~cf5d3hELN~8@~w?*;UW*|kflc*cE{+Rs0F5$%jrbwZMmYSW?B_9lGpDJ;5r;4QG zsitF8PYTJzPox$$vTh?qIIeiJo&GSmA-0Sm&PNo_wm=-Oq^Yr_=qIvjr`mm5r2_94 zc9SX}vC90_1D3p|+BC3L^AIF+e9ZeZF1gWYEhVrh?$1rws_W9?Lr%SX92#&U-=0oh z`#s$?Lvu1X9Gkki$YwV-y%w))AeI*?uyv$O4k)=Jqq$-G0L{%pK80mn1+q-ATs4LT zu;pg6<_j0NQ=_t_H;U~2b!&`+@Vc~_N*D>Zis*1k%Tv9Z>#lgrYpAol&1Ue5$-Mt#a+HAGMdZ50Slqlv} zf0fp+sc!^w%`n&kR!Zs$J=8?VE6gu2bzyV*tHZRYz==rL51F}$^W-L=y_;N;^p)~| z!&UwQM>TVKvg7LK!Db0vT7pwu`vl@T4gjYsdgG zY?@XQ^Q&oN!L&BH&#k8uC;KtAYni36r5uKlb+N6~{p+hUVP+fXHpCxQPa9Oij@T)p z1CT?l;nLMIM`EpcYz3>!`N-hW3~YL5ks+=8AGUm86;$kT$R)&9@oK_2RueGKk(&yN zJMMPP!lci;yD(q}Q!3Q;K_>GwHBV4LJj7O=JX4tBo4w*Zn{BchArg=yszP2x!KItY zE{i%4;ogLEwF^zG>OxL)2M>nMggIU@G1VgGzRg-)ubw#A{(}t$Js4XI4^3F20xv?5 ztm60&a^9H=Vk;B%@(Lo?R0o6yiN>@9rRh?t?h+@v6COZX*N`_1bO3g3^H^?Sbro_8 z*08=zJS*5kxs!B9+Umou7UH^KG#xw`7SV(!u^>8%>?vKhyc{;{{BS205Nytfm%I`1 z;bbRN%GwreZiG`(1E7weP~-F#SkhP{>bQ02s;rIi8Xo=|S3QjUq=S9nbcdBVht}-| zoKG>2+*%YV(VDYBe27b87R zd<;-j$W|9)$k!RQYpJh!#fo|{VRlpnIhP|ssC8I;TS4Fv3lMa_1}z5im-29ZIUGzl zNSL!(7hZW7UhTnTLc5-W(OrD_iW*hEHcR%4tIm_C&_YzGvjAQqt2z>-dN3SOAaz1@ zDR_}0C0+Lr*+=1Y+|k@mZfl-)lu9yR>n7#V>K|I+jTsS=aL}kYrNyrpdb6o}sCCH` zQhbT>MPS$|F?+7ugJU`ab1|25KS)a7CZa!<}z8WAt6-^}FD>XX+Br3>%gr#!?< z)M}%U7+t=~3h4tkEPK;JFe#nvL)GRD1qzeVfJ3F)q-$`nLxA0B)nYK(sLXnVz@Q~@ z3XZfSn@l9lnyrLhq@^1g&77OIwnR5F;+q5TTn5Ymvrrb>eO@mQ3E?FO-lalepyW#% zH^@v2REX2~{^WmK4_304x5Xlv!;8!Q+#%r~(o{A?^6I*Uxg!rMD;0zVEB6PdgMlfS zG#ik!A{bK5tuOvVQW^Q=p#iV591GfkOddYtgvA)i;f1c}MSNmR4xhjANKDP40n9|f zu_CERI<|rW9b5JX@}BrfWR)LhZOqc`dZUsl7cG>>;KUCS&cXPC@+>?8I&zyFI>N<( z1KlsY>(1@mid6vK6rg+aTbL2xOa%=^{U*9x0)e?ojZkL}%%z>B2o$l!aT%5)v@w!I z&sc?IKKw2)A9o@ivWjCCXdaDv%gkr!k0J6y3`}a{Ac?6Uo_K^jTzT@P(4r&<6hS)* zX(dUKNNEf?n1Y}p*pVxPumd_iW?`X@w}l1E0tM)XA!)go5)_ZABfFOO6mT!g`6B_A z;E_2f7N0;w9>wt6IBmiKBr$&QYiXk+_H6{mqk(j$-zZuzM|)CA9(DxRr0dTNCT z3W)~H=83J9_YwV5m`PAFvNl$DkR0 z0lwwP`o>@iAlEx~OL5WsmOSZ0Xt+d_ww6aTA+)p-7Iw%NHOA>O;^(JG#7eo_3p1R& zJbm+kRy$CuJ_T{oTgPZlT7(IbUqJXy5!)2p#=K}?y@C#Ae=QNJR!Ha!1V8sn&~TGFDyGDL=hGLI`B5n`{u|jPMln$Aw;%s~ z+$aX*5Q<{RvAd(d%ktH7)KDV%g@8p`RXbFkuPtY3jjLOuJ#knpW{ZV)@`u*gm>+p3 z2?E{FNto$QRV?_s3mS-EPmPHodz!6Vuy8Ku3W4{8I@EvbU|AF~Vkr(t^f4Ad$inZg zG%Mg^+TjwA))GHxsxt+Mks2;)2n|LAS(?Q`EZ@O77hwFz!56{(Q;|J9BvK+FrIL9- z5%Sy9XT#*!19-J|!sLNx$@T$6D}%#_xSIx+7WtzP_y#!_o&rTsG{xc#YKmy$w19OL z3U-i*ohZRuJak@z;Iji_BEHWP34=-<>tT-$*Eu4_5aTnmT^0vftLH%x-lFCqVxJp3 zu)r#D^g*6Aw}-z@VY0Xe&V=kwc0RE!6z(v772NmDE(D@SyB@Ku8YT&kj+4|Dpg%`L zIQ%g|qfmHb;O$~wl1-^neOdY$=r-z3Q7wuBL3;2ZAu3aXozx>2(m79T`V3~sz>By^ zWS6GfZDY-zc+A#){fNRz`wJ>>P`h_ko-_j1vE>i)#hAe0@?ot= zSYP9fa~YhOQ-!_8d;Kf@4xJ3U60;3R)@3JX69Kxsiv0u#hl{++62cG zv?Cfb6f~$pntfQnI(6(9NOIlLX|u?z*D|Hg-?WYaWP;YJ1&%P9sQCygx& z3#sf1oiAV;+RS!%_%J+Emxuk8!N4Mch++s?*5oNv%M4>$jcP5(HsPq670@0-8Ux!{ z@cMDCgT+M`o60#t4*T$F`(Ma7lLhCMnpdxeX9-(XmRKk>Tau^O_dwF|AOjENKyNS|^kdx)e`?%0@%JToWTzgzgJ&jqFuR@0$_%nOFDIdcs5e;fHU}+;evkYz1 zrLgVTN`oH~&q@rA){qb}jD>mlDoh1e##9A|mmA1jAB3$32O`c>ps@AZb)=~KK1boc zkcNj_9Fn`$?PF)vc+A zY_oHSxrQf8U1KI+XLgLYUGtHY{cd?tcDjf6UrS!RtJuz;AR%i3Ai+gE{+fWvW>hZG zt)CfKq^E0PLLW4xQB~^5I)Ic}RudBUSh|SLn6J2%F0EXPAw0<-og;p6kA9CvA88Nh$gZXFD2M5>V6x?x4um}TbCc-cyw)}4D{ zUj$Alg)_a}qpShlail5ZoCo zVsC++tvJ7c^Q9ZXb%VD^B!5JmYIr_5Jlxp1y*aV8An8WkVyo-!hGT|UDnj>hM&z6XsmM^8BuhU9Cq7D)yYGC4eta;$06PM{$@DrC8V9wCygJvKwBzCA9 zoiYP|8lSyHGtJ2y4+Ob;e7_Y8XFYfMgdHmLTkKGUa z*UfH1)nIA=O8k(6XjBp%am2|(pU~jX*CcJ$3F*UJQSg=-bCF!(;sx3Z2v?i=fKDKt%1`T+e4k)`x;jKFUj#8UV5hERxwJOqS zDrMKIAHWp(pwB@5_+NXnI0@KZ%q`?u0G5Eg=-nH27>6 z=YgeI-~Fzyq6H$>7f8+xzM(;fl0l>+Fpf`3KOIiNn$?_Jx~^7&H42potI{P%YwQ5w zC`{f*)W8Dt?IK{+wVbD88DgGfh-o$(<3^)s6(o=JY&;${HrwR!ZM6lKJ@F|dO(v9N z8*pum4!m596h*pcuu{t*qj=?M_>YD6NL7rxkEK>PMa2#lGBJ=4Zb6<)imXc1ayUZf zr1Ttk4Cn+yKNuiMPeefN>B(8$351xA?n+}GhcGcWn*Qn*=O}pDzNd&lC;FP=9DR*a z2rcN#OnA71B?OHE3+QY;X)j{s4o=#@x_dCv0NWw4bR#(t2~Gi|!)YA%rY0=5syoAR z&k*_e66WZLT2SdL{=wq{!Oyhi(c+7-=MKDpQ*?Mk6ges;7HKHhX(s8LjotJsHR96Qp4obg2GI~_z##uq?1WWVBvwiM)kCjBohj(h@GYK zAr6f%WP1xj1`89m!45T&?>trHtFN@#R=HDS?ZcJ=tqqHeHnDmtwS?0STTj!BViD`Q z!MuXcg?6BO1-6)zrWBlG&7KMEI%hE{BEZ&3r9Nbb=~)B*WUQe`QlV(P)9FleVI?T& zZw|)ZkReJO8_=HEt_;UbX{k<9r>1T_uD&eU*`U%rj66@6ry#sl#H>-vzUTB#=FihAvWYCC_F!<6sLs;kk^RRAz}t`_Nf~_CeVMA_tu?6QR@y(}6PguYK}u zY77yM4Gj-OX>vH3XxnWzALe-IRs!fDMGtGDCIyGOElCN$d+6gnh@oILm-hE8l$Hx< z@=|c-gT}`09QJNvj#|=^qEduSN;f`&u5Bs#1$JUzL zZhsRKueu;wA7|*qS4m)R;xpFbZN|y+td0<`gn29w(uIszGfULVRO7M%gD=`?2a@@N zt>0c1M_A$*ztGq}m1#Xqouw)NB;0m5SsI7t_-?zvqEFv;f((QjeV{iqjg#F_J7v2; zg{4LpQA^qZ8vd);Fr#hf5yOmTD%n`WL9qvM2t>pQf$|Nu)O<_cD{{w~mGN2Ri{0!b(^SM@}-0vJ;H5fbg)jU*B{y zr-=v-4cf%3`73n3VtMl6u!Ye3{ct}LKTUVPeNb9m>7vj?W_kJ2qzXbiqXMT-N$=T7 zzPxZ0ZxUw&mMkK>a7B<2UzhAJ9ztzrq}l3uBRGcsT2eMf@Nv=+I{DHWsjf7USgtrX zil5*_QMOWpH!hxGBVU;k=wI!6&l%y>krDhkvWRmkM(7J@&IsEHBltcuo$)f_Ex|`i zACXlb*m-1rKZI<~z`rv!yLagycD*$68lnyUo{5`t@z2tKT>r4^aF7^(pB=f+;{JON zJ=C)FU(-MAnpb*DcU~m@3vmAg{Im4KX9&?;?3% z*PRzCp2T`2Eos+FVX~7etVBy_ki{Xf%I>=l9M}jUDNbbe)?zS7b?eD z#2=KmM7`;{Fd#{;6d>H>zdxKU-_b6 zGtR=^Z}}aLq<_m8jRwcD>l68I@L#n4KaWTBe(OJHTGHF~aos^$`fE6R>)eg|VEXfZ zL(kaJNP4?ovtlXbYVBs|eUbEQFA@s7)}L&Ya*e+KlYnW6>acZ& zJV`%vlUz^Ke}9a$n*!;t(e&47ddUR@`m=kx9s^G8XYbEw`i!Rc8D7&z`~Mrji9`JK z4aqrkt6Yb*V`=Qt&)#eK{3Nj8`%5jMD7kWd7pEHhXX)+wy-0e;2FyO8xFOQumd>sZ zMbh8U({xq2UNd+#|7Gb6J_m&-%%py@t~Ckd*$aAElff M=gC%3zKf^-zm*fAAOHXW From 22de6c5c4cf466694ce755b8ff1c6b67aa27deac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Thu, 10 Aug 2023 18:09:49 +0300 Subject: [PATCH 088/242] upd .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index fd41f26cc0d20..93ce82ccee4c9 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ models-mnt /Pipfile /embd-input-test /gguf +/gguf-llama-simple /libllama.so build-info.h arm_neon.h From 42cc04d11d24b7b41e19ecaa38b46350faca141b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Thu, 10 Aug 2023 18:49:08 +0300 Subject: [PATCH 089/242] gguf : calculate n_mult --- gguf-llama.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index d385ef79a5707..b88a2d8bf9189 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -514,16 +514,30 @@ struct ggml_context * ctx_data = NULL; return gguf_get_arr_n(gguf_ctx, i); } + int find_n_mult(const int n_ff, const int n_embd) { + int n_mults[3] = {8192, 1, -1}; + for (int i = 0; i < 3; ++i) { + int calc_ff = (((8 * n_embd) / 3 + n_mults[i] - 1) / n_mults[i]) * n_mults[i]; + if (calc_ff == n_ff) { + return n_mults[i]; + } + } + + throw std::runtime_error(format("failed to find n_mult for n_ff = %d and n_emb = %d\n", n_ff, n_embd)); + } + void read_hparams() { // TODO make keysconstants in header // TODO: read all hparams from file hparams.n_vocab = read_n_vocab(); + hparams.n_ctx = read_u32("llama.context_length"); hparams.n_embd = read_u32("llama.embedding_length"); - //hparams.n_mult = file.read_u32(); + uint32_t n_ff = read_u32("llama.feed_forward_length"); + hparams.n_mult = find_n_mult(n_ff, hparams.n_embd); hparams.n_head = read_u32("llama.attention.head_count"); hparams.n_layer = read_u32("llama.layer_count"); - //hparams.n_rot = file.read_u32(); + hparams.n_rot = hparams.n_embd / hparams.n_head; //hparams.ftype = (enum llama_ftype) file.read_u32(); // LLaMAv2 @@ -568,7 +582,7 @@ struct ggml_context * ctx_data = NULL; for (uint32_t j = 0; j < n_dims; ++j) { tensor.ne[j] = cur->ne[j]; } - + if (n_dims < 1 || n_dims > 2) { throw std::runtime_error(format("llama.cpp: tensor '%s' should not be %u-dimensional", name, n_dims)); } From cfb8e35b73bd8daca6fa5179d30c48a5db43bc31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Thu, 10 Aug 2023 19:56:56 +0300 Subject: [PATCH 090/242] gguf : inference with 7B model working (WIP) --- gguf-llama.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index b88a2d8bf9189..0c40957142b41 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -493,6 +493,8 @@ struct ggml_context * ctx_data = NULL; gguf_ctx = gguf_init_from_file(fname, params); + read_hparams(); + read_vocab(); read_tensor_metadata(tensors_map); } @@ -523,7 +525,7 @@ struct ggml_context * ctx_data = NULL; } } - throw std::runtime_error(format("failed to find n_mult for n_ff = %d and n_emb = %d\n", n_ff, n_embd)); + throw std::runtime_error(format("failed to find n_mult for n_ff = %d and n_embd = %d\n", n_ff, n_embd)); } void read_hparams() { @@ -534,14 +536,14 @@ struct ggml_context * ctx_data = NULL; hparams.n_ctx = read_u32("llama.context_length"); hparams.n_embd = read_u32("llama.embedding_length"); uint32_t n_ff = read_u32("llama.feed_forward_length"); - hparams.n_mult = find_n_mult(n_ff, hparams.n_embd); + //hparams.n_mult = find_n_mult(n_ff, hparams.n_embd); hparams.n_head = read_u32("llama.attention.head_count"); hparams.n_layer = read_u32("llama.layer_count"); hparams.n_rot = hparams.n_embd / hparams.n_head; //hparams.ftype = (enum llama_ftype) file.read_u32(); // LLaMAv2 - hparams.n_head_kv = read_u32("llama.attention.head_count_kv"); + // hparams.n_head_kv = read_u32("llama.attention.head_count_kv"); } void read_vocab() { From f316b94c7c9a4f5a8cc5bc84f2fe7048a38b6ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Thu, 10 Aug 2023 20:20:22 +0300 Subject: [PATCH 091/242] gguf : rm deprecated function --- gguf-llama.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/gguf-llama.h b/gguf-llama.h index 20dcc9f639510..6062f375ea92c 100644 --- a/gguf-llama.h +++ b/gguf-llama.h @@ -222,13 +222,6 @@ extern "C" { struct llama_model * model, struct llama_context_params params); - // Various functions for loading a ggml llama model. - // Allocate (almost) all memory needed for the model. - // Return NULL on failure - LLAMA_API DEPRECATED(struct llama_context * llama_init_from_file( - const char * path_model, - struct llama_context_params params), - "please use llama_load_model_from_file combined with llama_new_context_with_model instead"); // Frees all allocated memory LLAMA_API void llama_free(struct llama_context * ctx); From e7d346c37c3ccd10baf1bc6895887a6ab17ee378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 09:52:01 +0300 Subject: [PATCH 092/242] gguf : start implementing gguf_file_saver (WIP) --- gguf-llama.cpp | 12 ++++++++++++ gguf-util.h | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 0c40957142b41..9c0b5651a0b13 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -626,20 +626,32 @@ struct gguf_file_saver { : file(fname, "wb"), any_file_loader(any_file_loader) { fprintf(stderr, "llama.cpp: saving model to %s\n", fname); write_magic(); + write_version(); write_hparams(new_ftype); write_vocab(); } + void write_magic() { + const int32_t magic = GGUF_MAGIC; + file.write_i32(magic); + } + + void write_version() { + const int32_t version = GGUF_VERSION; + file.write_i32(version); } + void write_hparams(enum llama_ftype new_ftype) { const llama_hparams & hparams = any_file_loader->hparams; GGML_UNUSED(hparams); GGML_UNUSED(new_ftype); } + void write_vocab() { uint32_t n_vocab = any_file_loader->hparams.n_vocab; GGML_UNUSED(n_vocab); } + void write_tensor(llama_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { switch (new_type) { case GGML_TYPE_F32: diff --git a/gguf-util.h b/gguf-util.h index 74d6e61f792d0..94ea9006adc84 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -61,6 +62,14 @@ static std::string format(const char * fmt, ...) { return std::string(buf.data(), size); } + +template +static std::string to_string(const T & val) { + std::stringstream ss; + ss << val; + return ss.str(); +} + // TODO: can we merge this one and gguf_context? struct gguf_file { // use FILE * so we don't have to re-open the file to mmap @@ -95,6 +104,42 @@ struct gguf_file { #endif GGML_ASSERT(ret == 0); // same } + + + void write_str(const std::string & val) { + const int32_t n = val.size(); + fwrite((const char *) &n, sizeof(n), 1, fp); + fwrite(val.c_str(), n, 1, fp); + } + + void write_i32(int32_t val) { + fwrite((const char *) &val, sizeof(val), 1, fp); + } + + void write_u64(size_t val) { + fwrite((const char *) &val, sizeof(val), 1, fp); + } + + template + void write_val(const std::string & key, enum gguf_type type, const T & val) { + write_str(key); + fwrite((const char *) &type, sizeof(type), 1, fp); + fwrite((const char *) &val, sizeof(val), 1, fp); + } + + template + void write_arr(const std::string & key, enum gguf_type type, const std::vector & val) { + write_str(key); + { + const enum gguf_type tarr = GGUF_TYPE_ARRAY; + fwrite((const char *) &tarr, sizeof(tarr), 1, fp); + } + + const int32_t n = val.size(); + fwrite((const char *) &type, sizeof(type), 1, fp); + fwrite((const char *) &n, sizeof(n), 1, fp); + fwrite(val.data(), sizeof(T), n, fp); + } }; #if defined(_WIN32) From a356b0e228b640a31cd9d1718d097a1aabcd7675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 10:50:02 +0300 Subject: [PATCH 093/242] gguf : start implementing gguf_file_saver (WIP) --- gguf-llama.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 9c0b5651a0b13..e88dc6a08510c 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -625,20 +625,25 @@ struct gguf_file_saver { gguf_file_saver(const char * fname, gguf_file_loader * any_file_loader, enum llama_ftype new_ftype) : file(fname, "wb"), any_file_loader(any_file_loader) { fprintf(stderr, "llama.cpp: saving model to %s\n", fname); - write_magic(); - write_version(); + write_header(); write_hparams(new_ftype); write_vocab(); } - void write_magic() { + // TODO: probably it's better to move these to gguf_file + + void write_header() { const int32_t magic = GGUF_MAGIC; file.write_i32(magic); - } - - void write_version() { + const int32_t version = GGUF_VERSION; file.write_i32(version); + + const int32_t n_tensors = gguf_get_n_tensors(any_file_loader->gguf_ctx); + file.write_i32(n_tensors); + + const int32_t n_kv = gguf_get_n_kv(any_file_loader->gguf_ctx); + file.write_i32(n_kv); } void write_hparams(enum llama_ftype new_ftype) { @@ -651,7 +656,7 @@ struct gguf_file_saver { uint32_t n_vocab = any_file_loader->hparams.n_vocab; GGML_UNUSED(n_vocab); } - + void write_tensor(llama_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { switch (new_type) { case GGML_TYPE_F32: From b2440f1943f88243938e438acef18ad072b9004e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 11:29:50 +0300 Subject: [PATCH 094/242] gguf : start implementing gguf_file_saver (WIP) --- gguf-llama.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index e88dc6a08510c..08b7004350ed7 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -647,9 +647,14 @@ struct gguf_file_saver { } void write_hparams(enum llama_ftype new_ftype) { - const llama_hparams & hparams = any_file_loader->hparams; - GGML_UNUSED(hparams); - GGML_UNUSED(new_ftype); + const int32_t n_kv = gguf_get_n_kv(any_file_loader->gguf_ctx); + for (int i = 0; i < n_kv; ++i) { + const char * key = gguf_get_key(any_file_loader->gguf_ctx, i); + if (strcmp(key, "general.quantization_version") == 0) { + file.write_val("general.quantization_version", GGUF_TYPE_UINT32, new_ftype); + } + } + } void write_vocab() { @@ -658,6 +663,10 @@ struct gguf_file_saver { } void write_tensor(llama_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { + GGML_UNUSED(tensor); + GGML_UNUSED(new_data); + GGML_UNUSED(new_size); + switch (new_type) { case GGML_TYPE_F32: case GGML_TYPE_F16: From eb8ca6996f5e3559505b3269d98e5d3ed860200e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 12:24:08 +0300 Subject: [PATCH 095/242] gguf : add gguf_get_kv_type --- ggml.c | 4 ++++ ggml.h | 9 +++++---- gguf-llama.cpp | 3 +++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ggml.c b/ggml.c index df8dce7e89915..ac45f12b08b13 100644 --- a/ggml.c +++ b/ggml.c @@ -19031,6 +19031,10 @@ const char * gguf_get_key(struct gguf_context * ctx, int i) { return ctx->header.kv[i].key.data; } +const enum gguf_type gguf_get_kv_type(struct gguf_context * ctx, int i) { + return ctx->header.kv[i].type; +} + const char * gguf_get_arr_str(struct gguf_context * ctx, int key_id, int i) { struct gguf_kv * kv = &ctx->header.kv[key_id]; struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[i]; diff --git a/ggml.h b/ggml.h index f1673eed52d20..4490e076cffcf 100644 --- a/ggml.h +++ b/ggml.h @@ -1744,10 +1744,11 @@ extern "C" { GGML_API size_t gguf_get_data_offset(struct gguf_context * ctx); GGML_API void * gguf_get_data (struct gguf_context * ctx); - GGML_API int gguf_get_n_kv(struct gguf_context * ctx); - GGML_API int gguf_find_key(struct gguf_context * ctx, const char * key); - GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); - GGML_API void gguf_get_val (struct gguf_context * ctx, int i, void * val); + GGML_API int gguf_get_n_kv(struct gguf_context * ctx); + GGML_API int gguf_find_key(struct gguf_context * ctx, const char * key); + GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); + GGML_API const enum gguf_type gguf_get_kv_type (struct gguf_context * ctx, int i); + GGML_API void gguf_get_val (struct gguf_context * ctx, int i, void * val); GGML_API const char * gguf_get_arr_str(struct gguf_context * ctx, int key_id, int i); GGML_API float gguf_get_arr_f32(struct gguf_context * ctx, int key_id, int i); diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 08b7004350ed7..8b928c364c766 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -652,6 +652,9 @@ struct gguf_file_saver { const char * key = gguf_get_key(any_file_loader->gguf_ctx, i); if (strcmp(key, "general.quantization_version") == 0) { file.write_val("general.quantization_version", GGUF_TYPE_UINT32, new_ftype); + } else { + const gguf_type vtype = gguf_get_kv_type(any_file_loader->gguf_ctx, i); + GGML_UNUSED(vtype); } } From e3a49609533646cbd1990f2d99668aef7a0cc420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 13:03:23 +0300 Subject: [PATCH 096/242] gguf : add gguf_get_kv_type --- ggml.c | 2 +- ggml.h | 10 +++++----- gguf-llama.cpp | 20 +++++++++++++++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/ggml.c b/ggml.c index ac45f12b08b13..e00f09fa458f7 100644 --- a/ggml.c +++ b/ggml.c @@ -19031,7 +19031,7 @@ const char * gguf_get_key(struct gguf_context * ctx, int i) { return ctx->header.kv[i].key.data; } -const enum gguf_type gguf_get_kv_type(struct gguf_context * ctx, int i) { +enum gguf_type gguf_get_kv_type(struct gguf_context * ctx, int i) { return ctx->header.kv[i].type; } diff --git a/ggml.h b/ggml.h index 4490e076cffcf..9a266e17573ce 100644 --- a/ggml.h +++ b/ggml.h @@ -1744,11 +1744,11 @@ extern "C" { GGML_API size_t gguf_get_data_offset(struct gguf_context * ctx); GGML_API void * gguf_get_data (struct gguf_context * ctx); - GGML_API int gguf_get_n_kv(struct gguf_context * ctx); - GGML_API int gguf_find_key(struct gguf_context * ctx, const char * key); - GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); - GGML_API const enum gguf_type gguf_get_kv_type (struct gguf_context * ctx, int i); - GGML_API void gguf_get_val (struct gguf_context * ctx, int i, void * val); + GGML_API int gguf_get_n_kv(struct gguf_context * ctx); + GGML_API int gguf_find_key(struct gguf_context * ctx, const char * key); + GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); + GGML_API enum gguf_type gguf_get_kv_type (struct gguf_context * ctx, int i); + GGML_API void gguf_get_val (struct gguf_context * ctx, int i, void * val); GGML_API const char * gguf_get_arr_str(struct gguf_context * ctx, int key_id, int i); GGML_API float gguf_get_arr_f32(struct gguf_context * ctx, int key_id, int i); diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 8b928c364c766..9ab7702774972 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -536,6 +536,7 @@ struct ggml_context * ctx_data = NULL; hparams.n_ctx = read_u32("llama.context_length"); hparams.n_embd = read_u32("llama.embedding_length"); uint32_t n_ff = read_u32("llama.feed_forward_length"); + GGML_UNUSED(n_ff); //hparams.n_mult = find_n_mult(n_ff, hparams.n_embd); hparams.n_head = read_u32("llama.attention.head_count"); hparams.n_layer = read_u32("llama.layer_count"); @@ -654,7 +655,21 @@ struct gguf_file_saver { file.write_val("general.quantization_version", GGUF_TYPE_UINT32, new_ftype); } else { const gguf_type vtype = gguf_get_kv_type(any_file_loader->gguf_ctx, i); - GGML_UNUSED(vtype); + switch(vtype) { + case GGUF_TYPE_BOOL: + case GGUF_TYPE_FLOAT32: + case GGUF_TYPE_INT16: + case GGUF_TYPE_INT32: + case GGUF_TYPE_INT8: + case GGUF_TYPE_STRING: + case GGUF_TYPE_UINT16: + case GGUF_TYPE_UINT32: + case GGUF_TYPE_UINT8: + case GGUF_TYPE_ARRAY: + break; + default: + throw std::runtime_error(format("cannot recognize value type for key %s\n", key)); + } } } @@ -3873,6 +3888,9 @@ bool llama_load_session_file(struct llama_context * ctx, const char * path_sessi bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { gguf_file file(path_session, "wb"); + GGML_UNUSED(ctx); + GGML_UNUSED(tokens); + GGML_UNUSED(n_token_count); // TODO: implement with GGUF format From 28abfc90fa9154be5ed9fded0870587aa6850820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 13:27:58 +0300 Subject: [PATCH 097/242] gguf : write metadata in gguf_file_saver (WIP) --- gguf-llama.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 9ab7702774972..11dcb1cc57ded 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -655,11 +655,29 @@ struct gguf_file_saver { file.write_val("general.quantization_version", GGUF_TYPE_UINT32, new_ftype); } else { const gguf_type vtype = gguf_get_kv_type(any_file_loader->gguf_ctx, i); + + bool bool_val; + float f32_val; + int16_t i16_val; + int32_t i32_val; + switch(vtype) { case GGUF_TYPE_BOOL: + bool_val = gguf_get_val_bool(any_file_loader->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_BOOL, bool_val); + break; case GGUF_TYPE_FLOAT32: + f32_val = gguf_get_val_f32(any_file_loader->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_FLOAT32, f32_val); + break; case GGUF_TYPE_INT16: + i16_val = gguf_get_val_i16(any_file_loader->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_INT16, i16_val); + break; case GGUF_TYPE_INT32: + i32_val = gguf_get_val_i32(any_file_loader->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_INT32, i32_val); + break; case GGUF_TYPE_INT8: case GGUF_TYPE_STRING: case GGUF_TYPE_UINT16: From 781b9ec3f504e1d306bcb0027dac65aa8e4dfc70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 18:01:26 +0300 Subject: [PATCH 098/242] gguf : write metadata in gguf_file_saver (WIP) --- gguf-llama.cpp | 21 +++++++++++++++++++++ gguf-util.h | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 11dcb1cc57ded..e70cae44c90a9 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -660,6 +660,12 @@ struct gguf_file_saver { float f32_val; int16_t i16_val; int32_t i32_val; + int8_t i8_val; + std::string str_val; + uint16_t u16_val; + uint32_t u32_val; + uint8_t u8_val; + switch(vtype) { case GGUF_TYPE_BOOL: @@ -679,10 +685,25 @@ struct gguf_file_saver { file.write_val(key, GGUF_TYPE_INT32, i32_val); break; case GGUF_TYPE_INT8: + i8_val = gguf_get_val_i8(any_file_loader->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_INT8, i8_val); + break; case GGUF_TYPE_STRING: + str_val = gguf_get_val_str(any_file_loader->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_STRING, str_val); + break; case GGUF_TYPE_UINT16: + u16_val = gguf_get_val_u16(any_file_loader->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_UINT16, u16_val); + break; case GGUF_TYPE_UINT32: + u32_val = gguf_get_val_u32(any_file_loader->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_UINT32, u32_val); + break; case GGUF_TYPE_UINT8: + u8_val = gguf_get_val_u8(any_file_loader->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_UINT8, u8_val); + break; case GGUF_TYPE_ARRAY: break; default: diff --git a/gguf-util.h b/gguf-util.h index 94ea9006adc84..9134019a45066 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -140,6 +140,33 @@ struct gguf_file { fwrite((const char *) &n, sizeof(n), 1, fp); fwrite(val.data(), sizeof(T), n, fp); } + template<> + void write_val(const std::string & key, enum gguf_type type, const std::string & val) { + write_str(key); + fwrite((const char *) &type, sizeof(type), 1, fp); + + const int32_t n = val.size(); + fwrite((const char *) &n, sizeof(n), 1, fp); + fwrite(val.c_str(), n, 1, fp); + } + + template<> + void write_arr(const std::string & key, enum gguf_type type, const std::vector & val) { + write_str(key); + { + const enum gguf_type tarr = GGUF_TYPE_ARRAY; + fwrite((const char *) &tarr, sizeof(tarr), 1, fp); + } + + const int32_t n = val.size(); + fwrite((const char *) &type, sizeof(type), 1, fp); + fwrite((const char *) &n, sizeof(n), 1, fp); + for (int i = 0; i < n; ++i) { + const int32_t nstr = val[i].size(); + fwrite((const char *) &nstr, sizeof(nstr), 1, fp); + fwrite(val[i].c_str(), nstr, 1, fp); + } + } }; #if defined(_WIN32) From d09fd107138d311151546f42a6881548ffd766c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 20:07:43 +0300 Subject: [PATCH 099/242] gguf : write metadata in gguf_file_saver --- ggml.c | 4 ++++ ggml.h | 1 + gguf-llama.cpp | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index e00f09fa458f7..c8fa60328e2d7 100644 --- a/ggml.c +++ b/ggml.c @@ -19035,6 +19035,10 @@ enum gguf_type gguf_get_kv_type(struct gguf_context * ctx, int i) { return ctx->header.kv[i].type; } +enum gguf_type gguf_get_arr_type(struct gguf_context * ctx, int i) { + return ctx->header.kv[i].value.arr.type; +} + const char * gguf_get_arr_str(struct gguf_context * ctx, int key_id, int i) { struct gguf_kv * kv = &ctx->header.kv[key_id]; struct gguf_str * str = &((struct gguf_str *) kv->value.arr.data)[i]; diff --git a/ggml.h b/ggml.h index 9a266e17573ce..fb3db10e2cedb 100644 --- a/ggml.h +++ b/ggml.h @@ -1748,6 +1748,7 @@ extern "C" { GGML_API int gguf_find_key(struct gguf_context * ctx, const char * key); GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i); GGML_API enum gguf_type gguf_get_kv_type (struct gguf_context * ctx, int i); + GGML_API enum gguf_type gguf_get_arr_type (struct gguf_context * ctx, int i); GGML_API void gguf_get_val (struct gguf_context * ctx, int i, void * val); GGML_API const char * gguf_get_arr_str(struct gguf_context * ctx, int key_id, int i); diff --git a/gguf-llama.cpp b/gguf-llama.cpp index e70cae44c90a9..27e0b5d43d875 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -647,6 +647,28 @@ struct gguf_file_saver { file.write_i32(n_kv); } + void write_hparam_arr_str(const std::string & key, enum gguf_type type, int i, int n_arr) { + std::vector data(n_arr); + + for (int j = 0; j < n_arr; ++j) { + std::string val = gguf_get_arr_str(any_file_loader->gguf_ctx, i, j); + data[j] = val; + } + + file.write_arr(key, type, data); + } + + void write_hparam_arr_f32(const std::string & key, enum gguf_type type, int i, int n_arr) { + std::vector data(n_arr); + + for (int j = 0; j < n_arr; ++j) { + float val = gguf_get_arr_f32(any_file_loader->gguf_ctx, i, j); + data[j] = val; + } + + file.write_arr(key, type, data); + } + void write_hparams(enum llama_ftype new_ftype) { const int32_t n_kv = gguf_get_n_kv(any_file_loader->gguf_ctx); for (int i = 0; i < n_kv; ++i) { @@ -665,7 +687,8 @@ struct gguf_file_saver { uint16_t u16_val; uint32_t u32_val; uint8_t u8_val; - + gguf_type arr_type; + int n_arr; switch(vtype) { case GGUF_TYPE_BOOL: @@ -705,6 +728,15 @@ struct gguf_file_saver { file.write_val(key, GGUF_TYPE_UINT8, u8_val); break; case GGUF_TYPE_ARRAY: + arr_type = gguf_get_arr_type(any_file_loader->gguf_ctx, i); + n_arr = gguf_get_arr_n(any_file_loader->gguf_ctx, i); + if (arr_type == GGUF_TYPE_FLOAT32) { + write_hparam_arr_f32(key, arr_type, i, n_arr); + } else if (arr_type == GGUF_TYPE_STRING) { + write_hparam_arr_str(key, GGUF_TYPE_STRING, i, n_arr); + } else { + throw std::runtime_error("not implemented"); + } break; default: throw std::runtime_error(format("cannot recognize value type for key %s\n", key)); From 61919c1a8f12cb14c41e73019180c5950c355c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 20:36:11 +0300 Subject: [PATCH 100/242] gguf : rm references to old file formats --- gguf-llama.cpp | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 27e0b5d43d875..7bd8cef6aee3e 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -467,13 +467,11 @@ struct llama_load_tensors_map { }; enum gguf_file_version { - gguf_file_VERSION_GGML, - gguf_file_VERSION_GGMF_V1, // added version field and scores in vocab - gguf_file_VERSION_GGJT_V1, // added padding - gguf_file_VERSION_GGJT_V2, // changed quantization format - gguf_file_VERSION_GGJT_V3, // changed Q4 and Q8 quantization format + GGUF_FILE_VERSION_V1 + }; + struct gguf_file_loader { gguf_file file; gguf_context * gguf_ctx; @@ -1069,12 +1067,8 @@ int64_t llama_time_us() { static const char *gguf_file_version_name(gguf_file_version version) { switch (version) { - case gguf_file_VERSION_GGML: return "'ggml' (old version with low tokenizer quality and no mmap support)"; - case gguf_file_VERSION_GGMF_V1: return "ggmf v1 (old version with no mmap support)"; - case gguf_file_VERSION_GGJT_V1: return "ggjt v1 (pre #1405)"; - case gguf_file_VERSION_GGJT_V2: return "ggjt v2 (pre #1508)"; - case gguf_file_VERSION_GGJT_V3: return "ggjt v3 (latest)"; - } + case GGUF_FILE_VERSION_V1: return "GGUF V1 (latest)"; + } return "unknown"; } @@ -1205,22 +1199,12 @@ static void llama_model_load_internal( fprintf(stderr, "%s: model size = %s\n", __func__, llama_model_type_name(model.type)); } - if (file_version < gguf_file_VERSION_GGJT_V2) { - if (hparams.ftype != LLAMA_FTYPE_ALL_F32 && - hparams.ftype != LLAMA_FTYPE_MOSTLY_F16 && - hparams.ftype != LLAMA_FTYPE_MOSTLY_Q8_0) { - throw std::runtime_error(format("this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1405)")); - } - } - - if (file_version < gguf_file_VERSION_GGJT_V3) { - if (hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || - hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_1 || - hparams.ftype == LLAMA_FTYPE_MOSTLY_Q8_0) { - throw std::runtime_error(format("this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1508)")); - } + if (hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || + hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_1 || + hparams.ftype == LLAMA_FTYPE_MOSTLY_Q8_0) { + throw std::runtime_error(format("this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1508)")); } - + if (vocab_only) { return; } From 7009cf581cbe958fcdeb94e58386646d0ebfa3dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 20:43:02 +0300 Subject: [PATCH 101/242] gguf : shorter name for member variable --- gguf-llama.cpp | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 7bd8cef6aee3e..77ba7d8806698 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -620,9 +620,9 @@ struct ggml_context * ctx_data = NULL; struct gguf_file_saver { gguf_file file; - gguf_file_loader * any_file_loader; - gguf_file_saver(const char * fname, gguf_file_loader * any_file_loader, enum llama_ftype new_ftype) - : file(fname, "wb"), any_file_loader(any_file_loader) { + gguf_file_loader * fl; + gguf_file_saver(const char * fname, gguf_file_loader * fl, enum llama_ftype new_ftype) + : file(fname, "wb"), fl(fl) { fprintf(stderr, "llama.cpp: saving model to %s\n", fname); write_header(); write_hparams(new_ftype); @@ -638,10 +638,10 @@ struct gguf_file_saver { const int32_t version = GGUF_VERSION; file.write_i32(version); - const int32_t n_tensors = gguf_get_n_tensors(any_file_loader->gguf_ctx); + const int32_t n_tensors = gguf_get_n_tensors(fl->gguf_ctx); file.write_i32(n_tensors); - const int32_t n_kv = gguf_get_n_kv(any_file_loader->gguf_ctx); + const int32_t n_kv = gguf_get_n_kv(fl->gguf_ctx); file.write_i32(n_kv); } @@ -649,7 +649,7 @@ struct gguf_file_saver { std::vector data(n_arr); for (int j = 0; j < n_arr; ++j) { - std::string val = gguf_get_arr_str(any_file_loader->gguf_ctx, i, j); + std::string val = gguf_get_arr_str(fl->gguf_ctx, i, j); data[j] = val; } @@ -660,7 +660,7 @@ struct gguf_file_saver { std::vector data(n_arr); for (int j = 0; j < n_arr; ++j) { - float val = gguf_get_arr_f32(any_file_loader->gguf_ctx, i, j); + float val = gguf_get_arr_f32(fl->gguf_ctx, i, j); data[j] = val; } @@ -668,13 +668,13 @@ struct gguf_file_saver { } void write_hparams(enum llama_ftype new_ftype) { - const int32_t n_kv = gguf_get_n_kv(any_file_loader->gguf_ctx); + const int32_t n_kv = gguf_get_n_kv(fl->gguf_ctx); for (int i = 0; i < n_kv; ++i) { - const char * key = gguf_get_key(any_file_loader->gguf_ctx, i); + const char * key = gguf_get_key(fl->gguf_ctx, i); if (strcmp(key, "general.quantization_version") == 0) { file.write_val("general.quantization_version", GGUF_TYPE_UINT32, new_ftype); } else { - const gguf_type vtype = gguf_get_kv_type(any_file_loader->gguf_ctx, i); + const gguf_type vtype = gguf_get_kv_type(fl->gguf_ctx, i); bool bool_val; float f32_val; @@ -690,44 +690,44 @@ struct gguf_file_saver { switch(vtype) { case GGUF_TYPE_BOOL: - bool_val = gguf_get_val_bool(any_file_loader->gguf_ctx, i); + bool_val = gguf_get_val_bool(fl->gguf_ctx, i); file.write_val(key, GGUF_TYPE_BOOL, bool_val); break; case GGUF_TYPE_FLOAT32: - f32_val = gguf_get_val_f32(any_file_loader->gguf_ctx, i); + f32_val = gguf_get_val_f32(fl->gguf_ctx, i); file.write_val(key, GGUF_TYPE_FLOAT32, f32_val); break; case GGUF_TYPE_INT16: - i16_val = gguf_get_val_i16(any_file_loader->gguf_ctx, i); + i16_val = gguf_get_val_i16(fl->gguf_ctx, i); file.write_val(key, GGUF_TYPE_INT16, i16_val); break; case GGUF_TYPE_INT32: - i32_val = gguf_get_val_i32(any_file_loader->gguf_ctx, i); + i32_val = gguf_get_val_i32(fl->gguf_ctx, i); file.write_val(key, GGUF_TYPE_INT32, i32_val); break; case GGUF_TYPE_INT8: - i8_val = gguf_get_val_i8(any_file_loader->gguf_ctx, i); + i8_val = gguf_get_val_i8(fl->gguf_ctx, i); file.write_val(key, GGUF_TYPE_INT8, i8_val); break; case GGUF_TYPE_STRING: - str_val = gguf_get_val_str(any_file_loader->gguf_ctx, i); + str_val = gguf_get_val_str(fl->gguf_ctx, i); file.write_val(key, GGUF_TYPE_STRING, str_val); break; case GGUF_TYPE_UINT16: - u16_val = gguf_get_val_u16(any_file_loader->gguf_ctx, i); + u16_val = gguf_get_val_u16(fl->gguf_ctx, i); file.write_val(key, GGUF_TYPE_UINT16, u16_val); break; case GGUF_TYPE_UINT32: - u32_val = gguf_get_val_u32(any_file_loader->gguf_ctx, i); + u32_val = gguf_get_val_u32(fl->gguf_ctx, i); file.write_val(key, GGUF_TYPE_UINT32, u32_val); break; case GGUF_TYPE_UINT8: - u8_val = gguf_get_val_u8(any_file_loader->gguf_ctx, i); + u8_val = gguf_get_val_u8(fl->gguf_ctx, i); file.write_val(key, GGUF_TYPE_UINT8, u8_val); break; case GGUF_TYPE_ARRAY: - arr_type = gguf_get_arr_type(any_file_loader->gguf_ctx, i); - n_arr = gguf_get_arr_n(any_file_loader->gguf_ctx, i); + arr_type = gguf_get_arr_type(fl->gguf_ctx, i); + n_arr = gguf_get_arr_n(fl->gguf_ctx, i); if (arr_type == GGUF_TYPE_FLOAT32) { write_hparam_arr_f32(key, arr_type, i, n_arr); } else if (arr_type == GGUF_TYPE_STRING) { @@ -745,7 +745,7 @@ struct gguf_file_saver { } void write_vocab() { - uint32_t n_vocab = any_file_loader->hparams.n_vocab; + uint32_t n_vocab = fl->hparams.n_vocab; GGML_UNUSED(n_vocab); } From f44bbd3d88e2cdb68e59c965e28961afe4a0ad26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 21:00:51 +0300 Subject: [PATCH 102/242] gguf : rm redundant method --- gguf-llama.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 77ba7d8806698..a8bd242b020e5 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -626,7 +626,6 @@ struct gguf_file_saver { fprintf(stderr, "llama.cpp: saving model to %s\n", fname); write_header(); write_hparams(new_ftype); - write_vocab(); } // TODO: probably it's better to move these to gguf_file @@ -744,10 +743,6 @@ struct gguf_file_saver { } - void write_vocab() { - uint32_t n_vocab = fl->hparams.n_vocab; - GGML_UNUSED(n_vocab); - } void write_tensor(llama_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { GGML_UNUSED(tensor); From e732423280ff72ece7bc8dfe8ec1fa7e3153714c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Fri, 11 Aug 2023 23:50:38 +0300 Subject: [PATCH 103/242] gguf : get rid of n_mult, read n_ff from file --- gguf-llama.cpp | 55 ++++++++++++++++++-------------------------------- 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index a8bd242b020e5..40d5ffd1469b2 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -177,15 +177,12 @@ struct llama_hparams { uint32_t n_vocab = 32000; uint32_t n_ctx = 512; // this is provided as user input? uint32_t n_embd = 4096; - uint32_t n_mult = 256; uint32_t n_head = 32; uint32_t n_head_kv = 32; uint32_t n_layer = 32; uint32_t n_rot = 64; + uint32_t n_ff = 11008; - // LLaMAv2 - // TODO: load from model data hparams - float f_ffn_mult = 1.0f; float f_rms_norm_eps = LLAMA_DEFAULT_RMS_EPS; float rope_freq_base = 10000.0f; @@ -467,7 +464,7 @@ struct llama_load_tensors_map { }; enum gguf_file_version { - GGUF_FILE_VERSION_V1 + GGUF_FILE_VERSION_V1 = 1, }; @@ -490,6 +487,7 @@ struct ggml_context * ctx_data = NULL; }; gguf_ctx = gguf_init_from_file(fname, params); + file_version = (enum gguf_file_version) gguf_get_version(gguf_ctx); read_hparams(); read_vocab(); @@ -505,6 +503,15 @@ struct ggml_context * ctx_data = NULL; return gguf_get_val_u32(gguf_ctx, i); } + float read_f32(const char * key) { + int i = gguf_find_key(gguf_ctx, key); + if (i == -1) { + throw std::runtime_error(format("cannot find param with key %s\n", key)); + } + + return gguf_get_val_f32(gguf_ctx, i); + } + int read_n_vocab() { int i = gguf_find_key(gguf_ctx, "tokenizer.ggml.tokens"); if (i == -1) { @@ -514,18 +521,6 @@ struct ggml_context * ctx_data = NULL; return gguf_get_arr_n(gguf_ctx, i); } - int find_n_mult(const int n_ff, const int n_embd) { - int n_mults[3] = {8192, 1, -1}; - for (int i = 0; i < 3; ++i) { - int calc_ff = (((8 * n_embd) / 3 + n_mults[i] - 1) / n_mults[i]) * n_mults[i]; - if (calc_ff == n_ff) { - return n_mults[i]; - } - } - - throw std::runtime_error(format("failed to find n_mult for n_ff = %d and n_embd = %d\n", n_ff, n_embd)); - } - void read_hparams() { // TODO make keysconstants in header @@ -533,14 +528,12 @@ struct ggml_context * ctx_data = NULL; hparams.n_vocab = read_n_vocab(); hparams.n_ctx = read_u32("llama.context_length"); hparams.n_embd = read_u32("llama.embedding_length"); - uint32_t n_ff = read_u32("llama.feed_forward_length"); - GGML_UNUSED(n_ff); - //hparams.n_mult = find_n_mult(n_ff, hparams.n_embd); + hparams.n_ff = read_u32("llama.feed_forward_length"); hparams.n_head = read_u32("llama.attention.head_count"); hparams.n_layer = read_u32("llama.layer_count"); - hparams.n_rot = hparams.n_embd / hparams.n_head; - //hparams.ftype = (enum llama_ftype) file.read_u32(); - + hparams.n_rot = read_u32("llama.rope.dimension_count"); + hparams.f_rms_norm_eps = read_f32("llama.attention.layer_norm_rms_epsilon"); + // LLaMAv2 // hparams.n_head_kv = read_u32("llama.attention.head_count_kv"); } @@ -1125,6 +1118,7 @@ static void llama_model_load_internal( bool vocab_only, llama_progress_callback progress_callback, void * progress_callback_user_data) { + GGML_UNUSED(rms_norm_eps); // TODO: update function signature to remove this model.t_start_us = ggml_time_us(); @@ -1137,9 +1131,6 @@ static void llama_model_load_internal( auto & hparams = model.hparams; - // TODO: read from file - hparams.f_rms_norm_eps = rms_norm_eps; - { switch (hparams.n_layer) { case 26: model.type = e_model::MODEL_3B; break; @@ -1162,25 +1153,19 @@ static void llama_model_load_internal( if (model.type == e_model::MODEL_65B && n_gqa == 8) { fprintf(stderr, "%s: warning: assuming 70B model based on GQA == %d\n", __func__, n_gqa); model.type = e_model::MODEL_70B; - hparams.f_ffn_mult = 1.3f; // from the params.json of the 70B model - } + } hparams.rope_freq_base = rope_freq_base; hparams.rope_freq_scale = rope_freq_scale; } - // ref: https://github.com/facebookresearch/llama/blob/6c7fe276574e78057f917549435a2554000a876d/llama/model.py#L194-L199 - const uint32_t n_ff_raw = 2*(4*hparams.n_embd)/3; - const uint32_t n_ff_mult = hparams.f_ffn_mult*n_ff_raw; - const uint32_t n_ff = ((n_ff_mult + hparams.n_mult - 1)/hparams.n_mult)*hparams.n_mult; - //const uint32_t n_ff = 28672; - + const uint32_t n_ff = hparams.n_ff; + { fprintf(stderr, "%s: format = %s\n", __func__, gguf_file_version_name(file_version)); fprintf(stderr, "%s: n_vocab = %u\n", __func__, hparams.n_vocab); fprintf(stderr, "%s: n_ctx = %u\n", __func__, hparams.n_ctx); fprintf(stderr, "%s: n_embd = %u\n", __func__, hparams.n_embd); - fprintf(stderr, "%s: n_mult = %u\n", __func__, hparams.n_mult); fprintf(stderr, "%s: n_head = %u\n", __func__, hparams.n_head); fprintf(stderr, "%s: n_head_kv = %u\n", __func__, hparams.n_head_kv); fprintf(stderr, "%s: n_layer = %u\n", __func__, hparams.n_layer); From 2a5ac7af44c258a725ae10180dc5993db972e527 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 11 Aug 2023 23:08:48 +0200 Subject: [PATCH 104/242] Update gguf_tensor_map.py --- gguf_tensor_map.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gguf_tensor_map.py b/gguf_tensor_map.py index 4fba633b2bf12..644c5914de9b2 100644 --- a/gguf_tensor_map.py +++ b/gguf_tensor_map.py @@ -26,15 +26,15 @@ def get_tensor_map( n_blocks : int): tensor_map["output"] = mapped_to # llama-pth # Attention and fee-forward layer blocks for i in range(0,n_blocks): - # Attention norm 1 - mapped_to = "transformer.blocks."+str(i)+".attn_norm_1" + # Attention norm + mapped_to = "transformer.blocks."+str(i)+".attn_norm" tensor_map["gpt_neox.layers."+str(i)+".input_layernorm"] = mapped_to # gptneox - tensor_map["transformer.h."+str(i)+".ln_1"] = mapped_to # gpt2 - tensor_map["transformer.blocks."+str(i)+".norm_1"] = mapped_to # mpt - tensor_map["transformer.h."+str(i)+".input_layernorm"] = mapped_to # falcon7b - tensor_map["transformer.h."+str(i)+".ln_attn"] = mapped_to # falcon40b - tensor_map["model.layers."+str(i)+".input_layernorm"] = mapped_to # llama-hf - tensor_map["layers."+str(i)+".attention_norm"] = mapped_to # llama-pth + tensor_map["transformer.h."+str(i)+".ln_1"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".norm_1"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".input_layernorm"] = mapped_to # falcon7b + tensor_map["transformer.h."+str(i)+".ln_attn"] = mapped_to # falcon40b + tensor_map["model.layers."+str(i)+".input_layernorm"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention_norm"] = mapped_to # llama-pth # Attention norm 2 mapped_to = "transformer.blocks."+str(i)+".attn_norm_2" tensor_map["transformer.h."+str(i)+".ln_mlp"] = mapped_to # falcon40b From e76c59d5245825ac1e1bf06bfc7f78ed1f6bcf16 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 11 Aug 2023 23:09:49 +0200 Subject: [PATCH 105/242] Update gptneox-main.cpp --- gptneox-main.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gptneox-main.cpp b/gptneox-main.cpp index 1667c4d545114..f2be93e4b1ae6 100644 --- a/gptneox-main.cpp +++ b/gptneox-main.cpp @@ -565,8 +565,8 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 std::string blocknamestart = "transformer.blocks." + std::to_string(i) + "."; - layer.ln_1_g = get_tensor_ex(ctx, blocknamestart + "attn_norm_1.weight" ); - layer.ln_1_b = get_tensor_ex(ctx, blocknamestart + "attn_norm_1.bias" ); + layer.ln_1_g = get_tensor_ex(ctx, blocknamestart + "attn_norm.weight" ); + layer.ln_1_b = get_tensor_ex(ctx, blocknamestart + "attn_norm.bias" ); layer.c_attn_attn_w = get_tensor_ex(ctx, blocknamestart + "attn_qkv.weight" ); layer.c_attn_attn_b = get_tensor_ex(ctx ,blocknamestart + "attn_qkv.bias" ); @@ -584,8 +584,8 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 layer.c_mlp_proj_b = get_tensor_ex(ctx, blocknamestart + "ffn_down.bias" ); // map by name - model.tensors[blocknamestart + "attn_norm_1.weight"] = layer.ln_1_g; - model.tensors[blocknamestart + "attn_norm_1.bias"] = layer.ln_1_b; + model.tensors[blocknamestart + "attn_norm.weight"] = layer.ln_1_g; + model.tensors[blocknamestart + "attn_norm.bias"] = layer.ln_1_b; model.tensors[blocknamestart + "attn_qkv.weight"] = layer.c_attn_attn_w; model.tensors[blocknamestart + "attn_qkv.bias"] = layer.c_attn_attn_b; From 2f52008b203fc97e72d850e02b08c0cd49f56969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 12 Aug 2023 07:24:46 +0300 Subject: [PATCH 106/242] gguf : rm references to old file magics --- gguf-llama.cpp | 4 ---- gguf-llama.h | 12 ------------ 2 files changed, 16 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 40d5ffd1469b2..f80f823a0d004 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -3367,10 +3367,6 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const { uint32_t magic; fin.read((char *) &magic, sizeof(magic)); - if (magic != LLAMA_FILE_MAGIC_GGLA) { - fprintf(stderr, "%s: bad file magic\n", __func__); - return 1; - } uint32_t format_version; fin.read((char *) &format_version, sizeof(format_version)); diff --git a/gguf-llama.h b/gguf-llama.h index 6062f375ea92c..d3c0d6b87f437 100644 --- a/gguf-llama.h +++ b/gguf-llama.h @@ -34,18 +34,6 @@ # define DEPRECATED(func, hint) func #endif -#define LLAMA_FILE_MAGIC_GGJT 0x67676a74u // 'ggjt' -#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' -#define LLAMA_FILE_MAGIC_GGMF 0x67676d66u // 'ggmf' -#define LLAMA_FILE_MAGIC_GGML 0x67676d6cu // 'ggml' -#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' - -#define LLAMA_FILE_VERSION 3 -#define LLAMA_FILE_MAGIC LLAMA_FILE_MAGIC_GGJT -#define LLAMA_FILE_MAGIC_UNVERSIONED LLAMA_FILE_MAGIC_GGML -#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN -#define LLAMA_SESSION_VERSION 1 - #define LLAMA_DEFAULT_SEED 0xFFFFFFFF #if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL) From 4fa017a1f9ed89c872ca007e0701f5d7ced4f8fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 12 Aug 2023 10:40:56 +0300 Subject: [PATCH 107/242] gguf : start implementing quantization (WIP) --- gguf-llama.cpp | 4 ++++ gguf-util.h | 23 ++++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index f80f823a0d004..0581755b14f64 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -614,6 +614,7 @@ struct ggml_context * ctx_data = NULL; struct gguf_file_saver { gguf_file file; gguf_file_loader * fl; + size_t info_offset; gguf_file_saver(const char * fname, gguf_file_loader * fl, enum llama_ftype new_ftype) : file(fname, "wb"), fl(fl) { fprintf(stderr, "llama.cpp: saving model to %s\n", fname); @@ -734,6 +735,9 @@ struct gguf_file_saver { } } + info_offset = file.tell(); + size_t count = gguf_get_data_offset(fl->gguf_ctx) - info_offset; + file.write_zeros(count); } diff --git a/gguf-util.h b/gguf-util.h index 9134019a45066..0964e6d02cc4f 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -106,18 +106,21 @@ struct gguf_file { } - void write_str(const std::string & val) { + size_t write_str(const std::string & val) { + size_t total_written = 0; const int32_t n = val.size(); - fwrite((const char *) &n, sizeof(n), 1, fp); - fwrite(val.c_str(), n, 1, fp); + total_written += fwrite((const char *) &n, sizeof(n), 1, fp); + total_written += fwrite(val.c_str(), n, 1, fp); + + return total_written; } - void write_i32(int32_t val) { - fwrite((const char *) &val, sizeof(val), 1, fp); + size_t write_i32(int32_t val) { + return fwrite((const char *) &val, sizeof(val), 1, fp); } - void write_u64(size_t val) { - fwrite((const char *) &val, sizeof(val), 1, fp); + size_t write_u64(size_t val) { + return fwrite((const char *) &val, sizeof(val), 1, fp); } template @@ -167,6 +170,12 @@ struct gguf_file { fwrite(val[i].c_str(), nstr, 1, fp); } } + + void write_zeros(size_t count) { + for (size_t i = 0; i < count; ++i) { + fputc(0, fp); + } + } }; #if defined(_WIN32) From 0e1a3c7e7dea7877840ed7a203d589bd841cadff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 12 Aug 2023 11:32:34 +0300 Subject: [PATCH 108/242] gguf : start implementing quantization (WIP) --- gguf-llama.cpp | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 0581755b14f64..cf2c56955e6b4 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -615,6 +615,8 @@ struct gguf_file_saver { gguf_file file; gguf_file_loader * fl; size_t info_offset; + size_t tensor_offset = 0; + gguf_file_saver(const char * fname, gguf_file_loader * fl, enum llama_ftype new_ftype) : file(fname, "wb"), fl(fl) { fprintf(stderr, "llama.cpp: saving model to %s\n", fname); @@ -622,8 +624,6 @@ struct gguf_file_saver { write_hparams(new_ftype); } - // TODO: probably it's better to move these to gguf_file - void write_header() { const int32_t magic = GGUF_MAGIC; file.write_i32(magic); @@ -740,12 +740,26 @@ struct gguf_file_saver { file.write_zeros(count); } + size_t write_tensor_info(llama_load_tensor & tensor) { + size_t total_written = 0; + file.seek(0, info_offset); + total_written += file.write_str(tensor.name); - void write_tensor(llama_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { - GGML_UNUSED(tensor); - GGML_UNUSED(new_data); - GGML_UNUSED(new_size); + int32_t n_dims = tensor.ne.size(); + file.write_i32(n_dims); + for (int32_t i = 0; i < n_dims; ++i) { + total_written += file.write_i32(i); + } + + total_written += file.write_u64(tensor_offset); + info_offset += total_written; + file.seek(0, SEEK_END); + + return total_written; + } + + void write_tensor(llama_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { switch (new_type) { case GGML_TYPE_F32: case GGML_TYPE_F16: @@ -763,6 +777,13 @@ struct gguf_file_saver { default: GGML_ASSERT(false); } + write_tensor_info(tensor); + // file.write_raw(new_data); + GGML_UNUSED(new_data); + size_t padded_size = GGML_PAD(new_size, GGUF_DEFAULT_ALIGNMENT); // TODO: handle custom alignment + size_t pad = padded_size - new_size; + file.write_zeros(pad); + tensor_offset += padded_size; } }; From c4f02b4f74c5fdb6493090c238891163709631e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 12 Aug 2023 12:01:17 +0300 Subject: [PATCH 109/242] gguf : start implementing quantization (WIP) --- gguf-llama.cpp | 3 +-- gguf-util.h | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index cf2c56955e6b4..defe26fe04ab8 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -778,8 +778,7 @@ struct gguf_file_saver { } write_tensor_info(tensor); - // file.write_raw(new_data); - GGML_UNUSED(new_data); + file.write_raw(new_data, new_size); size_t padded_size = GGML_PAD(new_size, GGUF_DEFAULT_ALIGNMENT); // TODO: handle custom alignment size_t pad = padded_size - new_size; file.write_zeros(pad); diff --git a/gguf-util.h b/gguf-util.h index 0964e6d02cc4f..17f9dc9680dc9 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -123,6 +123,10 @@ struct gguf_file { return fwrite((const char *) &val, sizeof(val), 1, fp); } + void write_raw(const void * data, size_t size) { + fwrite(data, size, 1, fp); + } + template void write_val(const std::string & key, enum gguf_type type, const T & val) { write_str(key); From b2571af255e6a1ed42dffeb74ef51390f2cf5144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 12 Aug 2023 14:28:17 +0300 Subject: [PATCH 110/242] gguf : start implementing quantization (WIP) --- Makefile | 2 +- examples/gguf/gguf.cpp | 10 +++++++--- gguf-llama.cpp | 8 ++++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index f5922c95d2f98..304b3035d64d0 100644 --- a/Makefile +++ b/Makefile @@ -393,7 +393,7 @@ $(LIB_PRE)embdinput$(DSO_EXT): examples/embd-input/embd-input.h examples/embd-in embd-input-test: $(LIB_PRE)embdinput$(DSO_EXT) examples/embd-input/embd-input-test.cpp build-info.h ggml.o llama.o common.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %$(DSO_EXT),$(filter-out %.h,$(filter-out %.hpp,$^))) -o $@ $(LDFLAGS) -L. -lembdinput -gguf: examples/gguf/gguf.cpp build-info.h ggml.o $(OBJS) +gguf: examples/gguf/gguf.cpp build-info.h ggml.o gguf-llama.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) gguf-llama-simple: examples/gguf/gguf-llama-simple.cpp build-info.h ggml.o gguf-llama.o common.o $(OBJS) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index a1b8edc71f373..6f454a2047ddd 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -1,5 +1,6 @@ #include "ggml.h" #include "gguf-util.h" +#include "gguf-llama.h" #include #include @@ -7,14 +8,14 @@ #include #include #include - +/* template static std::string to_string(const T & val) { std::stringstream ss; ss << val; return ss.str(); } - +*/ void gguf_ex_write_str(std::ofstream & fout, const std::string & val) { const int32_t n = val.size(); fout.write((const char *) &n, sizeof(n)); @@ -414,7 +415,7 @@ int main(int argc, char ** argv) { const std::string fname(argv[1]); const std::string mode (argv[2]); - GGML_ASSERT((mode == "r" || mode == "w") && "mode must be r or w"); + GGML_ASSERT((mode == "r" || mode == "w" || mode == "q") && "mode must be r, w or q"); if (mode == "w") { GGML_ASSERT(gguf_ex_write(fname) && "failed to write gguf file"); @@ -422,6 +423,9 @@ int main(int argc, char ** argv) { GGML_ASSERT(gguf_ex_read_0(fname) && "failed to read gguf file"); GGML_ASSERT(gguf_ex_read_1(fname) && "failed to read gguf file"); GGML_ASSERT(gguf_ex_read_2(fname) && "failed to read gguf file"); + } else if (mode == "q") { + llama_model_quantize_params params = llama_model_quantize_default_params(); + llama_model_quantize(fname.c_str(), "quant.gguf", ¶ms); } return 0; diff --git a/gguf-llama.cpp b/gguf-llama.cpp index defe26fe04ab8..f1755fef551bf 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -738,15 +738,19 @@ struct gguf_file_saver { info_offset = file.tell(); size_t count = gguf_get_data_offset(fl->gguf_ctx) - info_offset; file.write_zeros(count); + printf("info_offset = %zu\n", info_offset); + file.seek(info_offset, SEEK_SET); + GGML_ASSERT(info_offset == file.tell()); } size_t write_tensor_info(llama_load_tensor & tensor) { size_t total_written = 0; - file.seek(0, info_offset); + file.seek(info_offset, SEEK_SET); + GGML_ASSERT(info_offset == file.tell()); total_written += file.write_str(tensor.name); int32_t n_dims = tensor.ne.size(); - file.write_i32(n_dims); + total_written += file.write_i32(n_dims); for (int32_t i = 0; i < n_dims; ++i) { total_written += file.write_i32(i); } From fa7c39540cff0ce6f6d245baeb15ac1ebe6cdd69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 12 Aug 2023 15:55:58 +0300 Subject: [PATCH 111/242] gguf : start implementing quantization (WIP) --- gguf-llama.cpp | 15 +++++++++++---- gguf-util.h | 12 ++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index f1755fef551bf..eecefc0f6b1bd 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -525,6 +525,11 @@ struct ggml_context * ctx_data = NULL; // TODO make keysconstants in header // TODO: read all hparams from file + int q_ver_idx = gguf_find_key (gguf_ctx, "general.quantization_version"); + if (q_ver_idx != -1) { + hparams.ftype = gguf_get_val_u32(gguf_ctx, q_ver_idx); + } + hparams.n_vocab = read_n_vocab(); hparams.n_ctx = read_u32("llama.context_length"); hparams.n_embd = read_u32("llama.embedding_length"); @@ -738,27 +743,29 @@ struct gguf_file_saver { info_offset = file.tell(); size_t count = gguf_get_data_offset(fl->gguf_ctx) - info_offset; file.write_zeros(count); - printf("info_offset = %zu\n", info_offset); file.seek(info_offset, SEEK_SET); GGML_ASSERT(info_offset == file.tell()); } - size_t write_tensor_info(llama_load_tensor & tensor) { + size_t write_tensor_info(llama_load_tensor & tensor, enum ggml_type type) { size_t total_written = 0; file.seek(info_offset, SEEK_SET); GGML_ASSERT(info_offset == file.tell()); total_written += file.write_str(tensor.name); +printf("total_written = %zu, name = %s\n", total_written, tensor.name.c_str()); int32_t n_dims = tensor.ne.size(); total_written += file.write_i32(n_dims); for (int32_t i = 0; i < n_dims; ++i) { - total_written += file.write_i32(i); + total_written += file.write_i32(tensor.ne[i]); } + total_written += file.write_i32(type); total_written += file.write_u64(tensor_offset); info_offset += total_written; file.seek(0, SEEK_END); + printf("total_written = %zu\n", total_written); return total_written; } @@ -781,7 +788,7 @@ struct gguf_file_saver { default: GGML_ASSERT(false); } - write_tensor_info(tensor); + write_tensor_info(tensor, new_type); file.write_raw(new_data, new_size); size_t padded_size = GGML_PAD(new_size, GGUF_DEFAULT_ALIGNMENT); // TODO: handle custom alignment size_t pad = padded_size - new_size; diff --git a/gguf-util.h b/gguf-util.h index 17f9dc9680dc9..ed7d53f691891 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -109,18 +109,22 @@ struct gguf_file { size_t write_str(const std::string & val) { size_t total_written = 0; const int32_t n = val.size(); - total_written += fwrite((const char *) &n, sizeof(n), 1, fp); - total_written += fwrite(val.c_str(), n, 1, fp); + fwrite((const char *) &n, sizeof(n), 1, fp); + total_written += sizeof(n); + fwrite(val.c_str(), n, 1, fp); + total_written += n; return total_written; } size_t write_i32(int32_t val) { - return fwrite((const char *) &val, sizeof(val), 1, fp); + fwrite((const char *) &val, sizeof(val), 1, fp); + return sizeof(val); } size_t write_u64(size_t val) { - return fwrite((const char *) &val, sizeof(val), 1, fp); + fwrite((const char *) &val, sizeof(val), 1, fp); + return sizeof(val); } void write_raw(const void * data, size_t size) { From 1fc3d30b71a707187eb1f995c4776db7aaa6265a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 12 Aug 2023 16:09:47 +0300 Subject: [PATCH 112/242] gguf : start implementing quantization (WIP) --- examples/gguf/gguf.cpp | 2 +- gguf-llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index 6f454a2047ddd..08f2b6322041a 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -421,7 +421,7 @@ int main(int argc, char ** argv) { GGML_ASSERT(gguf_ex_write(fname) && "failed to write gguf file"); } else if (mode == "r") { GGML_ASSERT(gguf_ex_read_0(fname) && "failed to read gguf file"); - GGML_ASSERT(gguf_ex_read_1(fname) && "failed to read gguf file"); + //GGML_ASSERT(gguf_ex_read_1(fname) && "failed to read gguf file"); GGML_ASSERT(gguf_ex_read_2(fname) && "failed to read gguf file"); } else if (mode == "q") { llama_model_quantize_params params = llama_model_quantize_default_params(); diff --git a/gguf-llama.cpp b/gguf-llama.cpp index eecefc0f6b1bd..ea721a0c72858 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -527,7 +527,7 @@ struct ggml_context * ctx_data = NULL; // TODO: read all hparams from file int q_ver_idx = gguf_find_key (gguf_ctx, "general.quantization_version"); if (q_ver_idx != -1) { - hparams.ftype = gguf_get_val_u32(gguf_ctx, q_ver_idx); + hparams.ftype = (enum llama_ftype) gguf_get_val_u32(gguf_ctx, q_ver_idx); } hparams.n_vocab = read_n_vocab(); From 202eab04d3f5a06304b9fd43b4b6b079d3f76dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 12 Aug 2023 16:39:05 +0300 Subject: [PATCH 113/242] gguf : quantization is working --- examples/gguf/gguf.cpp | 2 +- gguf-llama.cpp | 7 ++----- gguf-util.h | 4 ++++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index 08f2b6322041a..6f454a2047ddd 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -421,7 +421,7 @@ int main(int argc, char ** argv) { GGML_ASSERT(gguf_ex_write(fname) && "failed to write gguf file"); } else if (mode == "r") { GGML_ASSERT(gguf_ex_read_0(fname) && "failed to read gguf file"); - //GGML_ASSERT(gguf_ex_read_1(fname) && "failed to read gguf file"); + GGML_ASSERT(gguf_ex_read_1(fname) && "failed to read gguf file"); GGML_ASSERT(gguf_ex_read_2(fname) && "failed to read gguf file"); } else if (mode == "q") { llama_model_quantize_params params = llama_model_quantize_default_params(); diff --git a/gguf-llama.cpp b/gguf-llama.cpp index ea721a0c72858..700d6009bcddb 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -752,7 +752,6 @@ struct gguf_file_saver { file.seek(info_offset, SEEK_SET); GGML_ASSERT(info_offset == file.tell()); total_written += file.write_str(tensor.name); -printf("total_written = %zu, name = %s\n", total_written, tensor.name.c_str()); int32_t n_dims = tensor.ne.size(); total_written += file.write_i32(n_dims); @@ -765,8 +764,7 @@ printf("total_written = %zu, name = %s\n", total_written, tensor.name.c_str()); info_offset += total_written; file.seek(0, SEEK_END); - printf("total_written = %zu\n", total_written); - + return total_written; } @@ -936,8 +934,7 @@ struct llama_model_loader { } else { gguf_file & file = file_loader->file; file.seek(lt.file_off, SEEK_SET); - // TODO - //file.read_raw(lt.data, lt.size); + file.read_raw(lt.data, lt.size); } if (0) { diff --git a/gguf-util.h b/gguf-util.h index ed7d53f691891..6395cf304e510 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -131,6 +131,10 @@ struct gguf_file { fwrite(data, size, 1, fp); } + void read_raw(void * data, size_t size) { + fread(data, size, 1, fp); + } + template void write_val(const std::string & key, enum gguf_type type, const T & val) { write_str(key); From 60d540831b626443caddad12e04792bce91d3b4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sat, 12 Aug 2023 21:42:31 +0300 Subject: [PATCH 114/242] gguf : roper closing of file --- gguf-util.h | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/gguf-util.h b/gguf-util.h index 6395cf304e510..70673404e6f0f 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -105,7 +105,6 @@ struct gguf_file { GGML_ASSERT(ret == 0); // same } - size_t write_str(const std::string & val) { size_t total_written = 0; const int32_t n = val.size(); @@ -127,14 +126,6 @@ struct gguf_file { return sizeof(val); } - void write_raw(const void * data, size_t size) { - fwrite(data, size, 1, fp); - } - - void read_raw(void * data, size_t size) { - fread(data, size, 1, fp); - } - template void write_val(const std::string & key, enum gguf_type type, const T & val) { write_str(key); @@ -155,6 +146,7 @@ struct gguf_file { fwrite((const char *) &n, sizeof(n), 1, fp); fwrite(val.data(), sizeof(T), n, fp); } + template<> void write_val(const std::string & key, enum gguf_type type, const std::string & val) { write_str(key); @@ -188,6 +180,37 @@ struct gguf_file { fputc(0, fp); } } + + void read_raw(void * ptr, size_t len) const { + if (len == 0) { + return; + } + errno = 0; + std::size_t ret = std::fread(ptr, len, 1, fp); + if (ferror(fp)) { + throw std::runtime_error(format("read error: %s", strerror(errno))); + } + if (ret != 1) { + throw std::runtime_error(std::string("unexpectedly reached end of file")); + } + } + + void write_raw(const void * ptr, size_t len) const { + if (len == 0) { + return; + } + errno = 0; + size_t ret = std::fwrite(ptr, len, 1, fp); + if (ret != 1) { + throw std::runtime_error(format("write error: %s", strerror(errno))); + } + } + + ~gguf_file() { + if (fp) { + std::fclose(fp); + } + } }; #if defined(_WIN32) From 5d81a715d4a8e1e6d9dad2d978510d997d20595c Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sat, 12 Aug 2023 21:45:45 +0200 Subject: [PATCH 115/242] gguf.py : no need to convert tensors twice --- gguf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gguf.py b/gguf.py index 5eb21ee05fb6f..0854418a6d842 100644 --- a/gguf.py +++ b/gguf.py @@ -179,20 +179,20 @@ def add_val(self: str, val: Any, vtype: GGUFValueType = None, add_vtype: bool = def ggml_pad(x: int, n: int) -> int: return ((x + n - 1) // n) * n - def add_tensor_info(self, name: str, tensor: np.ndarray): + def add_tensor_info(self, name: str, tensor_shape: np.ndarray, tensor_dtype: np.dtype, tensor_nbytes: int): encoded_name = name.encode("utf8") self.ti_data += struct.pack(" Date: Sat, 12 Aug 2023 21:48:58 +0200 Subject: [PATCH 116/242] convert-gptneox-h5-to-gguf.py : no need to convert tensors twice --- convert-gptneox-h5-to-gguf.py | 65 ++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index 22508bd3dae56..0c0cd6bb13c73 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -43,7 +43,6 @@ def bytes_to_unicode(): # output in the same directory as the model dir_model = sys.argv[1] -fname_out = sys.argv[1] + "/ggml-model.bin" last_dir = os.path.basename(os.path.normpath(dir_model)) # possible tensor data types @@ -59,7 +58,8 @@ def bytes_to_unicode(): if ftype < 0 or ftype > 1: print("Invalid ftype: " + str(ftype)) sys.exit(1) - fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" + +fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" print("gguf: loading model "+last_dir) @@ -82,7 +82,6 @@ def bytes_to_unicode(): block_count = hparams["num_hidden_layers"] gguf_writer.add_name(last_dir) -gguf_writer.add_description("gguf test model") gguf_writer.add_architecture(llm_arch) gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) @@ -201,22 +200,30 @@ def bytes_to_unicode(): sys.exit() n_dims = len(data.shape) + data_dtype = data.dtype - # ftype == 0 -> float32, ftype == 1 -> float16 - ftype_cur = 0 - if ftype != 0: - if name.endswith(".weight") and n_dims == 2: - data = data.astype(np.float16) - ftype_cur = 1 - else: - data = data.astype(np.float32) - ftype_cur = 0 - else: - if data.dtype != np.float32: - data = data.astype(np.float32) - ftype_cur = 0 +# print( name + " dims " + str(n_dims) + " dtype " + str(data.dtype) ) + + if data.dtype != np.float16 and data.dtype != np.float32: + # convert any unsupported data types to float32 + data_dtype = np.float32 + elif ftype == 1 and data.dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + # if f16 desired, convert any float32 2-dim weight tensors to float16 + data_dtype = np.float16 + + nelements = 1 + + for i in range(n_dims): + nelements *= data.shape[n_dims - 1 - i] - gguf_writer.add_tensor_info(name, data) + data_nbytes = 0 + if data_dtype == np.float16: + data_nbytes = nelements * 2 + elif data_dtype == np.float32: + data_nbytes = nelements * 4 + + + gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes) print("gguf: write header") gguf_writer.write_header_to_file() @@ -226,7 +233,7 @@ def bytes_to_unicode(): gguf_writer.write_ti_data_to_file() # tensor data -print("gguf: write tensor data") +print("gguf: convert and write tensor data") for name in list_vars.keys(): data = list_vars[name].squeeze().numpy() @@ -236,20 +243,14 @@ def bytes_to_unicode(): continue n_dims = len(data.shape) - - # ftype == 0 -> float32, ftype == 1 -> float16 - ftype_cur = 0 - if ftype != 0: - if name.endswith(".weight") and n_dims == 2: - data = data.astype(np.float16) - ftype_cur = 1 - else: - data = data.astype(np.float32) - ftype_cur = 0 - else: - if data.dtype != np.float32: - data = data.astype(np.float32) - ftype_cur = 0 + data_dtype = data.dtype + + if data_dtype != np.float16 and data_dtype != np.float32: + # convert any unsupported data types to float32 + data = data.astype(np.float32) + elif ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + # if f16 desired, convert any float32 2-dim weight tensors to float16 + data = data.astype(np.float16) gguf_writer.write_tensor_to_file(data) From 4cef57c81a2bc7afff72869e6b9659177a11d334 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sat, 12 Aug 2023 21:50:24 +0200 Subject: [PATCH 117/242] convert-llama-h5-to-gguf.py : no need to convert tensors twice --- convert-llama-h5-to-gguf.py | 67 +++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 0b477a1336570..519b73966fa3a 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -32,7 +32,6 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # output in the same directory as the model dir_model = sys.argv[1] -fname_out = sys.argv[1] + "/ggml-model.bin" last_dir = os.path.basename(os.path.normpath(dir_model)) @@ -49,7 +48,8 @@ def permute(weights: NDArray, n_head: int) -> NDArray: if ftype < 0 or ftype > 1: print("Invalid ftype: " + str(ftype)) sys.exit(1) - fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" + +fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" print("gguf: loading model "+last_dir) @@ -72,8 +72,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: head_count = hparams["num_attention_heads"] block_count = hparams["num_hidden_layers"] -gguf_writer.add_name("llama2-7b") -gguf_writer.add_description("gguf test model") +gguf_writer.add_name(last_dir) gguf_writer.add_architecture(llm_arch) gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) @@ -186,22 +185,30 @@ def permute(weights: NDArray, n_head: int) -> NDArray: sys.exit() n_dims = len(data.shape) + data_dtype = data.dtype - # ftype == 0 -> float32, ftype == 1 -> float16 - ftype_cur = 0 - if ftype != 0: - if name.endswith(".weight") and n_dims == 2: - data = data.astype(np.float16) - ftype_cur = 1 - else: - data = data.astype(np.float32) - ftype_cur = 0 - else: - if data.dtype != np.float32: - data = data.astype(np.float32) - ftype_cur = 0 +# print( name + " dims " + str(n_dims) + " dtype " + str(data.dtype) ) + + if data.dtype != np.float16 and data.dtype != np.float32: + # convert any unsupported data types to float32 + data_dtype = np.float32 + elif ftype == 1 and data.dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + # if f16 desired, convert any float32 2-dim weight tensors to float16 + data_dtype = np.float16 + + nelements = 1 + + for i in range(n_dims): + nelements *= data.shape[n_dims - 1 - i] - gguf_writer.add_tensor_info(name, data) + data_nbytes = 0 + if data_dtype == np.float16: + data_nbytes = nelements * 2 + elif data_dtype == np.float32: + data_nbytes = nelements * 4 + + + gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes) print("gguf: write header") @@ -212,7 +219,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: gguf_writer.write_ti_data_to_file() # tensor data -print("gguf: write tensor data") +print("gguf: convert and write tensor data") for name in list_vars.keys(): data = list_vars[name].squeeze().numpy() @@ -226,20 +233,14 @@ def permute(weights: NDArray, n_head: int) -> NDArray: data = permute(data, head_count) n_dims = len(data.shape) - - # ftype == 0 -> float32, ftype == 1 -> float16 - ftype_cur = 0 - if ftype != 0: - if name.endswith(".weight") and n_dims == 2: - data = data.astype(np.float16) - ftype_cur = 1 - else: - data = data.astype(np.float32) - ftype_cur = 0 - else: - if data.dtype != np.float32: - data = data.astype(np.float32) - ftype_cur = 0 + data_dtype = data.dtype + + if data_dtype != np.float16 and data_dtype != np.float32: + # convert any unsupported data types to float32 + data = data.astype(np.float32) + elif ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + # if f16 desired, convert any float32 2-dim weight tensors to float16 + data = data.astype(np.float16) gguf_writer.write_tensor_to_file(data) From f8218477b37d14b522e6422bdbb93302d8b8f421 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sat, 12 Aug 2023 22:29:35 +0200 Subject: [PATCH 118/242] convert-gptneox-h5-to-gguf.py : simplify nbytes --- convert-gptneox-h5-to-gguf.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index 0c0cd6bb13c73..770a5d9caa313 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -211,17 +211,7 @@ def bytes_to_unicode(): # if f16 desired, convert any float32 2-dim weight tensors to float16 data_dtype = np.float16 - nelements = 1 - - for i in range(n_dims): - nelements *= data.shape[n_dims - 1 - i] - - data_nbytes = 0 - if data_dtype == np.float16: - data_nbytes = nelements * 2 - elif data_dtype == np.float32: - data_nbytes = nelements * 4 - + data_nbytes = data.size * 2 if data_dtype == np.float16 else data.size * 4 gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes) From e606ffeaeed3bca9d5a1d97774eaf1a3d602088f Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sat, 12 Aug 2023 22:30:35 +0200 Subject: [PATCH 119/242] convert-llama-h5-to-gguf.py : simplify nbytes --- convert-llama-h5-to-gguf.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 519b73966fa3a..bf6ff6aa78edd 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -196,17 +196,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # if f16 desired, convert any float32 2-dim weight tensors to float16 data_dtype = np.float16 - nelements = 1 - - for i in range(n_dims): - nelements *= data.shape[n_dims - 1 - i] - - data_nbytes = 0 - if data_dtype == np.float16: - data_nbytes = nelements * 2 - elif data_dtype == np.float32: - data_nbytes = nelements * 4 - + data_nbytes = data.size * 2 if data_dtype == np.float16 else data.size * 4 gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes) From 5e58ffa1ed8a97297429367947206dbece94e30d Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sat, 12 Aug 2023 23:50:58 +0200 Subject: [PATCH 120/242] gptneox-main.cpp : n_layer --> n_block --- gptneox-main.cpp | 135 +++++++++++++++++++---------------------------- 1 file changed, 53 insertions(+), 82 deletions(-) diff --git a/gptneox-main.cpp b/gptneox-main.cpp index f2be93e4b1ae6..7420daf967399 100644 --- a/gptneox-main.cpp +++ b/gptneox-main.cpp @@ -24,13 +24,13 @@ struct gpt_neox_hparams { uint32_t n_ctx = 0; uint32_t n_embd = 0; uint32_t n_head = 0; - uint32_t n_layer = 0; + uint32_t n_block = 0; uint32_t n_rot = 0; // rotary_pct * (n_embd / n_head) bool par_res = true; float norm_eps = 1e-5; }; -struct gpt_neox_layer { +struct gpt_neox_block { // pre normalization struct ggml_tensor * ln_1_g; struct ggml_tensor * ln_1_b; @@ -65,7 +65,7 @@ struct gpt_neox_model { struct ggml_tensor * lmh_g; // language model head - std::vector layers; + std::vector blocks; // key + value memory struct ggml_tensor * memory_k; @@ -415,7 +415,7 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 if (keyidx != -1) { hparams.n_head = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.layer_count"); - if (keyidx != -1) { hparams.n_layer = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } + if (keyidx != -1) { hparams.n_block = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.rope.dimension_count"); if (keyidx != -1) { hparams.n_rot = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } @@ -434,7 +434,7 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx); printf("%s: n_embd = %d\n", __func__, hparams.n_embd); printf("%s: n_head = %d\n", __func__, hparams.n_head); - printf("%s: n_layer = %d\n", __func__, hparams.n_layer); + printf("%s: n_block = %d\n", __func__, hparams.n_block); printf("%s: n_rot = %d\n", __func__, hparams.n_rot); printf("%s: par_res = %d\n", __func__, hparams.par_res); printf("%s: norm_eps = %g\n", __func__, hparams.norm_eps); @@ -545,9 +545,9 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 // prepare memory for the weights { - const int n_layer = model.hparams.n_layer; + const int n_block = model.hparams.n_block; - model.layers.resize(n_layer); + model.blocks.resize(n_block); model.wte = ggml_get_tensor(ctx, "transformer.token_embd.weight"); model.ln_f_g = ggml_get_tensor(ctx, "transformer.output_norm.weight"); @@ -560,47 +560,47 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 model.tensors["transformer.output_norm.bias"] = model.ln_f_b; model.tensors["transformer.output.weight"] = model.lmh_g; - for (int i = 0; i < n_layer; ++i) { - auto & layer = model.layers[i]; + for (int i = 0; i < n_block; ++i) { + auto & block = model.blocks[i]; std::string blocknamestart = "transformer.blocks." + std::to_string(i) + "."; - layer.ln_1_g = get_tensor_ex(ctx, blocknamestart + "attn_norm.weight" ); - layer.ln_1_b = get_tensor_ex(ctx, blocknamestart + "attn_norm.bias" ); + block.ln_1_g = get_tensor_ex(ctx, blocknamestart + "attn_norm.weight" ); + block.ln_1_b = get_tensor_ex(ctx, blocknamestart + "attn_norm.bias" ); - layer.c_attn_attn_w = get_tensor_ex(ctx, blocknamestart + "attn_qkv.weight" ); - layer.c_attn_attn_b = get_tensor_ex(ctx ,blocknamestart + "attn_qkv.bias" ); + block.c_attn_attn_w = get_tensor_ex(ctx, blocknamestart + "attn_qkv.weight" ); + block.c_attn_attn_b = get_tensor_ex(ctx ,blocknamestart + "attn_qkv.bias" ); - layer.c_attn_proj_w = get_tensor_ex(ctx, blocknamestart + "attn_output.weight" ); - layer.c_attn_proj_b = get_tensor_ex(ctx, blocknamestart + "attn_output.bias" ); + block.c_attn_proj_w = get_tensor_ex(ctx, blocknamestart + "attn_output.weight" ); + block.c_attn_proj_b = get_tensor_ex(ctx, blocknamestart + "attn_output.bias" ); - layer.ln_2_g = get_tensor_ex(ctx, blocknamestart + "ffn_norm.weight" ); - layer.ln_2_b = get_tensor_ex(ctx, blocknamestart + "ffn_norm.bias"); + block.ln_2_g = get_tensor_ex(ctx, blocknamestart + "ffn_norm.weight" ); + block.ln_2_b = get_tensor_ex(ctx, blocknamestart + "ffn_norm.bias"); - layer.c_mlp_fc_w = get_tensor_ex(ctx, blocknamestart + "ffn_up.weight" ); - layer.c_mlp_fc_b = get_tensor_ex(ctx, blocknamestart + "ffn_up.bias" ); + block.c_mlp_fc_w = get_tensor_ex(ctx, blocknamestart + "ffn_up.weight" ); + block.c_mlp_fc_b = get_tensor_ex(ctx, blocknamestart + "ffn_up.bias" ); - layer.c_mlp_proj_w = get_tensor_ex(ctx, blocknamestart + "ffn_down.weight" ); - layer.c_mlp_proj_b = get_tensor_ex(ctx, blocknamestart + "ffn_down.bias" ); + block.c_mlp_proj_w = get_tensor_ex(ctx, blocknamestart + "ffn_down.weight" ); + block.c_mlp_proj_b = get_tensor_ex(ctx, blocknamestart + "ffn_down.bias" ); // map by name - model.tensors[blocknamestart + "attn_norm.weight"] = layer.ln_1_g; - model.tensors[blocknamestart + "attn_norm.bias"] = layer.ln_1_b; + model.tensors[blocknamestart + "attn_norm.weight"] = block.ln_1_g; + model.tensors[blocknamestart + "attn_norm.bias"] = block.ln_1_b; - model.tensors[blocknamestart + "attn_qkv.weight"] = layer.c_attn_attn_w; - model.tensors[blocknamestart + "attn_qkv.bias"] = layer.c_attn_attn_b; + model.tensors[blocknamestart + "attn_qkv.weight"] = block.c_attn_attn_w; + model.tensors[blocknamestart + "attn_qkv.bias"] = block.c_attn_attn_b; - model.tensors[blocknamestart + "attn_output.weight"] = layer.c_attn_proj_w; - model.tensors[blocknamestart + "attn_output.bias"] = layer.c_attn_proj_b; + model.tensors[blocknamestart + "attn_output.weight"] = block.c_attn_proj_w; + model.tensors[blocknamestart + "attn_output.bias"] = block.c_attn_proj_b; - model.tensors[blocknamestart + "ffn_norm.weight"] = layer.ln_2_g; - model.tensors[blocknamestart + "ffn_norm.bias"] = layer.ln_2_b; + model.tensors[blocknamestart + "ffn_norm.weight"] = block.ln_2_g; + model.tensors[blocknamestart + "ffn_norm.bias"] = block.ln_2_b; - model.tensors[blocknamestart + "ffn_up.weight"] = layer.c_mlp_fc_w; - model.tensors[blocknamestart + "ffn_up.bias"] = layer.c_mlp_fc_b; + model.tensors[blocknamestart + "ffn_up.weight"] = block.c_mlp_fc_w; + model.tensors[blocknamestart + "ffn_up.bias"] = block.c_mlp_fc_b; - model.tensors[blocknamestart + "ffn_down.weight"] = layer.c_mlp_proj_w; - model.tensors[blocknamestart + "ffn_down.bias"] = layer.c_mlp_proj_b; + model.tensors[blocknamestart + "ffn_down.weight"] = block.c_mlp_proj_w; + model.tensors[blocknamestart + "ffn_down.bias"] = block.c_mlp_proj_b; } } @@ -610,10 +610,10 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 const auto & hparams = model.hparams; const int n_embd = hparams.n_embd; - const int n_layer = hparams.n_layer; + const int n_block = hparams.n_block; const int n_ctx = hparams.n_ctx; - const int64_t n_mem = n_layer*n_ctx; + const int64_t n_mem = n_block*n_ctx; const int64_t n_elements = n_embd*n_mem; // create the ggml context @@ -647,37 +647,23 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 // feed-forward network ggml_tensor * gpt_neox_ff( - const gpt_neox_layer &layer, + const gpt_neox_block &block, ggml_context * ctx0, ggml_tensor * inp) { ggml_tensor * cur = ggml_norm(ctx0, inp); - cur = ggml_add(ctx0, - ggml_mul(ctx0, - ggml_repeat(ctx0, layer.ln_2_g, cur), - cur), - ggml_repeat(ctx0, layer.ln_2_b, cur)); - - cur = ggml_mul_mat(ctx0, - layer.c_mlp_fc_w, - cur); - - cur = ggml_add(ctx0, - ggml_repeat(ctx0, layer.c_mlp_fc_b, cur), - cur); + cur = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, block.ln_2_g, cur), cur), ggml_repeat(ctx0, block.ln_2_b, cur)); + cur = ggml_mul_mat(ctx0, block.c_mlp_fc_w, cur); + cur = ggml_add(ctx0, ggml_repeat(ctx0, block.c_mlp_fc_b, cur), cur); // GELU activation cur = ggml_gelu(ctx0, cur); // projection // cur = proj_w*cur + proj_b - cur = ggml_mul_mat(ctx0, - layer.c_mlp_proj_w, - cur); + cur = ggml_mul_mat(ctx0, block.c_mlp_proj_w, cur); - cur = ggml_add(ctx0, - ggml_repeat(ctx0, layer.c_mlp_proj_b, cur), - cur); + cur = ggml_add(ctx0, ggml_repeat(ctx0, block.c_mlp_proj_b, cur), cur); return cur; } @@ -701,7 +687,7 @@ bool gpt_neox_eval( const auto & hparams = model.hparams; const int n_embd = hparams.n_embd; - const int n_layer = hparams.n_layer; + const int n_block = hparams.n_block; const int n_ctx = hparams.n_ctx; const int n_head = hparams.n_head; const int n_vocab = hparams.n_vocab; @@ -747,7 +733,7 @@ bool gpt_neox_eval( // wte struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd); - for (int il = 0; il < n_layer; ++il) { + for (int il = 0; il < n_block; ++il) { struct ggml_tensor * cur; ggml_set_scratch(ctx0, { 0, scr0_size, scr0, }); @@ -758,22 +744,15 @@ bool gpt_neox_eval( cur = ggml_norm(ctx0, inpL); cur = ggml_add(ctx0, - ggml_mul(ctx0, - ggml_repeat(ctx0, model.layers[il].ln_1_g, cur), - cur), - ggml_repeat(ctx0, model.layers[il].ln_1_b, cur)); + ggml_mul(ctx0, ggml_repeat(ctx0, model.blocks[il].ln_1_g, cur), cur), + ggml_repeat(ctx0, model.blocks[il].ln_1_b, cur)); } // compute QKV { - cur = ggml_mul_mat(ctx0, - model.layers[il].c_attn_attn_w, - cur); - - cur = ggml_add(ctx0, - ggml_repeat(ctx0, model.layers[il].c_attn_attn_b, cur), - cur); + cur = ggml_mul_mat(ctx0, model.blocks[il].c_attn_attn_w, cur); + cur = ggml_add(ctx0, ggml_repeat(ctx0, model.blocks[il].c_attn_attn_b, cur), cur); } struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_3d(ctx0, cur, n_embd/n_head, n_head, N, cur->nb[1]/n_head, cur->nb[1], 0*sizeof(float)*n_embd/n_head)); @@ -798,10 +777,7 @@ bool gpt_neox_eval( } // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3) - struct ggml_tensor * Q = - ggml_permute(ctx0, - Qcur, - 0, 2, 1, 3); + struct ggml_tensor * Q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3); // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3) struct ggml_tensor * K = @@ -842,17 +818,12 @@ bool gpt_neox_eval( struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3); // cur = KQV_merged.contiguous().view(n_embd, N) - cur = ggml_cpy(ctx0, - KQV_merged, - ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N)); + cur = ggml_cpy(ctx0, KQV_merged, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N)); // projection { - cur = ggml_mul_mat(ctx0, - model.layers[il].c_attn_proj_w, - cur); - - cur = ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].c_attn_proj_b, cur), cur); + cur = ggml_mul_mat(ctx0, model.blocks[il].c_attn_proj_w, cur); + cur = ggml_add(ctx0, ggml_repeat(ctx0, model.blocks[il].c_attn_proj_b, cur), cur); } } @@ -861,7 +832,7 @@ bool gpt_neox_eval( if (hparams.par_res == 0) { struct ggml_tensor * inpFF = ggml_add(ctx0, cur, inpL); - cur = gpt_neox_ff(model.layers[il], ctx0, inpFF); + cur = gpt_neox_ff(model.blocks[il], ctx0, inpFF); // input for next layer inpL = ggml_add(ctx0, cur, inpFF); @@ -870,7 +841,7 @@ bool gpt_neox_eval( // this is independent of the self-attention result, so it could be done in parallel to the self-attention // note here we pass inpL instead of cur - cur = gpt_neox_ff(model.layers[il], ctx0, inpL); + cur = gpt_neox_ff(model.blocks[il], ctx0, inpL); // layer input + FF cur = ggml_add(ctx0, cur, inpFF); From 8b5f0c506708b06fde1362fc2cbb464222cf44aa Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 13 Aug 2023 00:00:32 +0200 Subject: [PATCH 121/242] constants.py : n_layer --> n_block --- constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constants.py b/constants.py index ae6e719fb555e..7fa238a73278c 100644 --- a/constants.py +++ b/constants.py @@ -18,7 +18,7 @@ # LLM KEY_LLM_CONTEXT_LENGTH = "{llm}.context_length" KEY_LLM_EMBEDDING_LENGTH = "{llm}.embedding_length" -KEY_LLM_LAYER_COUNT = "{llm}.layer_count" +KEY_LLM_BLOCK_COUNT = "{llm}.block_count" KEY_LLM_FEED_FORWARD_LENGTH = "{llm}.feed_forward_length" KEY_LLM_USE_PARALLEL_RESIDUAL = "{llm}.use_parallel_residual" KEY_LLM_TENSOR_DATA_LAYOUT = "{llm}.tensor_data_layout" From d2ce9cfe8d0c75ef4166bac28e93854319219390 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 13 Aug 2023 00:01:20 +0200 Subject: [PATCH 122/242] gguf.py : n_layer --> n_block --- gguf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gguf.py b/gguf.py index 0854418a6d842..75ba2870afc9a 100644 --- a/gguf.py +++ b/gguf.py @@ -253,9 +253,9 @@ def add_embedding_length(self, llm: str, length: int): self.add_uint32( constants.KEY_LLM_EMBEDDING_LENGTH.format(llm=llm), length) - def add_layer_count(self, llm: str, length: int): + def add_block_count(self, llm: str, length: int): self.add_uint32( - constants.KEY_LLM_LAYER_COUNT.format(llm=llm), length) + constants.KEY_LLM_BLOCK_COUNT.format(llm=llm), length) def add_feed_forward_length(self, llm: str, length: int): self.add_uint32( From 489616e12652235ecb5c7f3839a2f5e6b617bb82 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 13 Aug 2023 00:02:04 +0200 Subject: [PATCH 123/242] convert-gptneox-h5-to-gguf.py : n_layer --> n_block --- convert-gptneox-h5-to-gguf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index 770a5d9caa313..ac4b2ff8860fd 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -85,7 +85,7 @@ def bytes_to_unicode(): gguf_writer.add_architecture(llm_arch) gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) -gguf_writer.add_layer_count(llm_arch, block_count) +gguf_writer.add_block_count(llm_arch, block_count) gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"]) gguf_writer.add_rope_dimension_count(llm_arch, int( hparams["rotary_pct"]*(hparams["hidden_size"]//hparams["num_attention_heads"])) ) gguf_writer.add_head_count(llm_arch, hparams["num_attention_heads"]) @@ -116,7 +116,7 @@ def bytes_to_unicode(): vocab_size = len( tokenizer_json["model"]["vocab"] ) - # from ggllm.cpp falcon_convert.py + # ref: https://github.com/cmp-nct/ggllm.cpp/blob/master/falcon_convert.py tokenizer = AutoTokenizer.from_pretrained(dir_model) reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()} From e91a2224e49aeed6e1382643dcb053ec6c168440 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 13 Aug 2023 00:02:44 +0200 Subject: [PATCH 124/242] convert-llama-h5-to-gguf.py : n_layer --> n_block --- convert-llama-h5-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index bf6ff6aa78edd..055b6b78da9c2 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -76,7 +76,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: gguf_writer.add_architecture(llm_arch) gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) -gguf_writer.add_layer_count(llm_arch, block_count) +gguf_writer.add_block_count(llm_arch, block_count) gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"]) gguf_writer.add_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"]) gguf_writer.add_head_count(llm_arch, head_count) From c7bd8c147ccd7411d1cd9a2a394d2d9b743d7294 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 13 Aug 2023 00:03:32 +0200 Subject: [PATCH 125/242] gptneox-main.cpp : n_layer --> n_block --- gptneox-main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gptneox-main.cpp b/gptneox-main.cpp index 7420daf967399..4773dfd690461 100644 --- a/gptneox-main.cpp +++ b/gptneox-main.cpp @@ -414,7 +414,7 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.attention.head_count"); if (keyidx != -1) { hparams.n_head = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } - if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.layer_count"); + if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.block_count"); if (keyidx != -1) { hparams.n_block = gguf_get_val_u32(ggufctx, keyidx); } else { ok = false; } } if (ok) { keyidx = gguf_find_key(ggufctx, "gptneox.rope.dimension_count"); From 9bf5a7efcb55fb96c08e1fab44e2ebe61964dba6 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 13 Aug 2023 01:27:38 +0200 Subject: [PATCH 126/242] Update gguf_tensor_map.py --- gguf_tensor_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf_tensor_map.py b/gguf_tensor_map.py index 644c5914de9b2..d73788bb46877 100644 --- a/gguf_tensor_map.py +++ b/gguf_tensor_map.py @@ -68,7 +68,7 @@ def get_tensor_map( n_blocks : int): mapped_to = "transformer.blocks."+str(i)+".ffn_norm" tensor_map["gpt_neox.layers."+str(i)+".post_attention_layernorm"] = mapped_to # gptneox tensor_map["transformer.h."+str(i)+".ln_2"] = mapped_to # gpt2 - tensor_map[" transformer.blocks."+str(i)+".norm_2"] = mapped_to # mpt + tensor_map["transformer.blocks."+str(i)+".norm_2"] = mapped_to # mpt tensor_map["model.layers."+str(i)+".post_attention_layernorm"] = mapped_to # llama-hf tensor_map["layers."+str(i)+".ffn_norm"] = mapped_to # llama-pth # Feed-forward up From e3d1f07eb1d18e2545a5f1a2bafc86b1fd1db053 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 13 Aug 2023 12:18:34 +0200 Subject: [PATCH 127/242] convert-gptneox-h5-to-gguf.py : load model in parts to save memory --- convert-gptneox-h5-to-gguf.py | 145 ++++++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 40 deletions(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index ac4b2ff8860fd..a8a0c7e2df357 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -1,4 +1,4 @@ -# Quick and dirty HF gptneox--> gguf conversion +# HF gptneox--> gguf conversion import gguf import gguf_tensor_map as tmap @@ -9,7 +9,8 @@ import numpy as np from typing import Any, List from pathlib import Path -from transformers import AutoTokenizer, AutoModelForCausalLM +import torch +from transformers import AutoTokenizer # ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py def bytes_to_unicode(): @@ -33,6 +34,15 @@ def bytes_to_unicode(): cs = [chr(n) for n in cs] return dict(zip(bs, cs)) +def count_model_parts(dir_model: str) -> int: + num_parts = 0 + for filename in os.listdir(dir_model): + if filename.startswith("pytorch_model-"): + num_parts += 1 + + if num_parts > 0: + print("gguf: found " + str(num_parts) + " model parts") + return num_parts if len(sys.argv) < 3: print("Usage: convert-h5-to-ggml.py dir-model ftype\n") @@ -70,9 +80,8 @@ def bytes_to_unicode(): print("Model architecture not supported: " + hparams["architectures"][0] ) sys.exit() - -model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) -list_vars = model.state_dict() +# get number of model parts +num_parts = count_model_parts(dir_model) gguf_writer = gguf.GGUFWriter.open(fname_out) @@ -183,37 +192,58 @@ def bytes_to_unicode(): # tensor info print("gguf: get tensor metadata") -for name in list_vars.keys(): - data = list_vars[name].squeeze().numpy() - - # we don't need these - if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): - continue +if num_parts == 0: + part_names = ("pytorch_model.bin",) +else: + part_names = ( + f"pytorch_model-{n:05}-of-{num_parts:05}.bin" for n in range(1, num_parts + 1) + ) - # map tensor names - if name.endswith(".weight") and name[:-7] in tensor_map: - name = tensor_map[name[:-7]] + ".weight" - elif name.endswith(".bias") and name[:-5] in tensor_map: - name = tensor_map[name[:-5]] + ".bias" - else: - print( "Can not map tensor '" + name + "'" ) - sys.exit() +for part_name in part_names: + print("gguf: loading model part '"+ part_name + "'") + model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu") - n_dims = len(data.shape) - data_dtype = data.dtype + for name in model_part.keys(): + data = model_part[name] -# print( name + " dims " + str(n_dims) + " dtype " + str(data.dtype) ) + # we don't need these + if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): + continue - if data.dtype != np.float16 and data.dtype != np.float32: # convert any unsupported data types to float32 - data_dtype = np.float32 - elif ftype == 1 and data.dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + if data.dtype != torch.float16 and data.dtype != torch.float32: + data = data.to(torch.float32) + + data = data.squeeze().numpy() + + # map tensor names + if name.endswith(".weight") and name[:-7] in tensor_map: + name = tensor_map[name[:-7]] + ".weight" + elif name.endswith(".bias") and name[:-5] in tensor_map: + name = tensor_map[name[:-5]] + ".bias" + else: + print( "Can not map tensor '" + name + "'" ) + sys.exit() + + n_dims = len(data.shape) + data_dtype = data.dtype + + # if f32 desired, convert any float16 to float32 + if ftype == 0 and data.dtype == np.float16: + data_dtype = np.float32 + + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if ftype == 1 and data.dtype == np.float16 and n_dims == 1: + data_dtype = np.float32 + # if f16 desired, convert any float32 2-dim weight tensors to float16 - data_dtype = np.float16 + if ftype == 1 and data.dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data_dtype = np.float16 + + data_nbytes = data.size * 2 if data_dtype == np.float16 else data.size * 4 - data_nbytes = data.size * 2 if data_dtype == np.float16 else data.size * 4 + gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes) - gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes) print("gguf: write header") gguf_writer.write_header_to_file() @@ -225,24 +255,59 @@ def bytes_to_unicode(): # tensor data print("gguf: convert and write tensor data") -for name in list_vars.keys(): - data = list_vars[name].squeeze().numpy() +if num_parts == 0: + part_names = ("pytorch_model.bin",) +else: + part_names = ( + f"pytorch_model-{n:05}-of-{num_parts:05}.bin" for n in range(1, num_parts + 1) + ) - # we don't need these - if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): - continue +for part_name in part_names: + print("gguf: loading model part '"+ part_name + "'") + model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu") - n_dims = len(data.shape) - data_dtype = data.dtype + for name in model_part.keys(): + data = model_part[name] + + old_dtype = data.dtype + + # we don't need these + if name.endswith(".attention.masked_bias") or name.endswith(".attention.bias") or name.endswith(".attention.rotary_emb.inv_freq"): + continue - if data_dtype != np.float16 and data_dtype != np.float32: # convert any unsupported data types to float32 - data = data.astype(np.float32) - elif ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + if data.dtype != torch.float16 and data.dtype != torch.float32: + data = data.to(torch.float32) + + data = data.squeeze().numpy() + + # map tensor names + if name.endswith(".weight") and name[:-7] in tensor_map: + name = tensor_map[name[:-7]] + ".weight" + elif name.endswith(".bias") and name[:-5] in tensor_map: + name = tensor_map[name[:-5]] + ".bias" + else: + print( "Can not map tensor '" + name + "'" ) + sys.exit() + + n_dims = len(data.shape) + data_dtype = data.dtype + + # if f32 desired, convert any float16 to float32 + if ftype == 0 and data.dtype == np.float16: + data = data.astype(np.float32) + + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if ftype == 1 and data_dtype == np.float16 and n_dims == 1: + data = data.astype(np.float32) + # if f16 desired, convert any float32 2-dim weight tensors to float16 - data = data.astype(np.float16) + if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data = data.astype(np.float16) + + print( name + ", shape " + str(len(data.shape)) + ", " + str(old_dtype) + " --> " + str(data.dtype)) - gguf_writer.write_tensor_to_file(data) + gguf_writer.write_tensor_to_file(data) gguf_writer.close() From 17800cd80fec468411481dc34a51d42a936442f1 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 13 Aug 2023 12:20:02 +0200 Subject: [PATCH 128/242] convert-llama-h5-to-gguf.py : load model in parts to save memory --- convert-llama-h5-to-gguf.py | 155 +++++++++++++++++++++++++----------- 1 file changed, 110 insertions(+), 45 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 055b6b78da9c2..98a14db9cf3c4 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -1,4 +1,4 @@ -# Quick and dirty HF llama --> gguf conversion, GQA/70b wont work +# HF llama --> gguf conversion, GQA/70b not supported import gguf import gguf_tensor_map as tmap @@ -9,7 +9,7 @@ import numpy as np from typing import Any, List from pathlib import Path -from transformers import AutoModelForCausalLM +import torch from sentencepiece import SentencePieceProcessor @@ -22,6 +22,15 @@ def permute(weights: NDArray, n_head: int) -> NDArray: .swapaxes(1, 2) .reshape(weights.shape)) +def count_model_parts(dir_model: str) -> int: + num_parts = 0 + for filename in os.listdir(dir_model): + if filename.startswith("pytorch_model-"): + num_parts += 1 + + if num_parts > 0: + print("gguf: found " + str(num_parts) + " model parts") + return num_parts if len(sys.argv) < 3: print("Usage: convert-h5-to-ggml.py dir-model ftype\n") @@ -60,8 +69,8 @@ def permute(weights: NDArray, n_head: int) -> NDArray: print("Model architecture not supported: " + hparams["architectures"][0] ) sys.exit() -model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) -list_vars = model.state_dict() +# get number of model parts +num_parts = count_model_parts(dir_model) gguf_writer = gguf.GGUFWriter.open(fname_out) @@ -164,41 +173,62 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # tensor info print("gguf: get tensor metadata") -for name in list_vars.keys(): - data = list_vars[name].squeeze().numpy() - - # we don't need these - if name.endswith(".rotary_emb.inv_freq"): - continue - # permute these - if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): - data = permute(data,head_count) +if num_parts == 0: + part_names = ("pytorch_model.bin",) +else: + part_names = ( + f"pytorch_model-{n:05}-of-{num_parts:05}.bin" for n in range(1, num_parts + 1) + ) - # map tensor names - if name.endswith(".weight") and name[:-7] in tensor_map: - name = tensor_map[name[:-7]] + ".weight" - elif name.endswith(".bias") and name[:-5] in tensor_map: - name = tensor_map[name[:-5]] + ".bias" - else: - print( "Can not map tensor '" + name + "'" ) - sys.exit() +for part_name in part_names: + print("gguf: loading model part '"+ part_name + "'") + model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu") - n_dims = len(data.shape) - data_dtype = data.dtype + for name in model_part.keys(): + data = model_part[name] -# print( name + " dims " + str(n_dims) + " dtype " + str(data.dtype) ) + # we don't need these + if name.endswith(".rotary_emb.inv_freq"): + continue - if data.dtype != np.float16 and data.dtype != np.float32: # convert any unsupported data types to float32 - data_dtype = np.float32 - elif ftype == 1 and data.dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + if data.dtype != torch.float16 and data.dtype != torch.float32: + data = data.to(torch.float32) + + data = data.squeeze().numpy() + + # permute these + if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): + data = permute(data,head_count) + + # map tensor names + if name.endswith(".weight") and name[:-7] in tensor_map: + name = tensor_map[name[:-7]] + ".weight" + elif name.endswith(".bias") and name[:-5] in tensor_map: + name = tensor_map[name[:-5]] + ".bias" + else: + print( "Can not map tensor '" + name + "'" ) + sys.exit() + + n_dims = len(data.shape) + data_dtype = data.dtype + + # if f32 desired, convert any float16 to float32 + if ftype == 0 and data.dtype == np.float16: + data_dtype = np.float32 + + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if ftype == 1 and data_dtype == np.float16 and n_dims == 1: + data_dtype = np.float32 + # if f16 desired, convert any float32 2-dim weight tensors to float16 - data_dtype = np.float16 + if ftype == 1 and data.dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data_dtype = np.float16 - data_nbytes = data.size * 2 if data_dtype == np.float16 else data.size * 4 + data_nbytes = data.size * 2 if data_dtype == np.float16 else data.size * 4 - gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes) + gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes) print("gguf: write header") @@ -211,28 +241,63 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # tensor data print("gguf: convert and write tensor data") -for name in list_vars.keys(): - data = list_vars[name].squeeze().numpy() +if num_parts == 0: + part_names = ("pytorch_model.bin",) +else: + part_names = ( + f"pytorch_model-{n:05}-of-{num_parts:05}.bin" for n in range(1, num_parts + 1) + ) - # we don't need these - if name.endswith(".rotary_emb.inv_freq"): - continue +for part_name in part_names: + print("gguf: loading model part '"+ part_name + "'") + model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu") - # permute these - if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): - data = permute(data, head_count) + for name in model_part.keys(): + data = model_part[name] - n_dims = len(data.shape) - data_dtype = data.dtype + old_dtype = data.dtype + + # we don't need these + if name.endswith(".rotary_emb.inv_freq"): + continue - if data_dtype != np.float16 and data_dtype != np.float32: # convert any unsupported data types to float32 - data = data.astype(np.float32) - elif ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + if data.dtype != torch.float16 and data.dtype != torch.float32: + data = data.to(torch.float32) + + data = data.squeeze().numpy() + + # permute these + if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): + data = permute(data, head_count) + + # map tensor names + if name.endswith(".weight") and name[:-7] in tensor_map: + name = tensor_map[name[:-7]] + ".weight" + elif name.endswith(".bias") and name[:-5] in tensor_map: + name = tensor_map[name[:-5]] + ".bias" + else: + print( "Can not map tensor '" + name + "'" ) + sys.exit() + + n_dims = len(data.shape) + data_dtype = data.dtype + + # if f32 desired, convert any float16 to float32 + if ftype == 0 and data.dtype == np.float16: + data = data.astype(np.float32) + + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if ftype == 1 and data_dtype == np.float16 and n_dims == 1: + data = data.astype(np.float32) + # if f16 desired, convert any float32 2-dim weight tensors to float16 - data = data.astype(np.float16) + if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data = data.astype(np.float16) + + print( name + ", shape " + str(len(data.shape)) + ", " + str(old_dtype) + " --> " + str(data.dtype)) - gguf_writer.write_tensor_to_file(data) + gguf_writer.write_tensor_to_file(data) gguf_writer.close() From 91d4bfd536cee322c26a3d70ac41d486a3fbb7b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sun, 13 Aug 2023 13:29:46 +0300 Subject: [PATCH 129/242] convert : write more metadata for LLaMA --- convert-llama-h5-to-gguf.py | 24 +++++++++++++++--------- gguf.py | 23 +++++------------------ 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 055b6b78da9c2..9526bde842fa5 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -17,6 +17,7 @@ # compatible with python < 3.9 NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]' + def permute(weights: NDArray, n_head: int) -> NDArray: return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) .swapaxes(1, 2) @@ -52,12 +53,12 @@ def permute(weights: NDArray, n_head: int) -> NDArray: fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" print("gguf: loading model "+last_dir) - + with open(dir_model + "/config.json", "r", encoding="utf-8") as f: hparams = json.load(f) if hparams["architectures"][0] != "LlamaForCausalLM": - print("Model architecture not supported: " + hparams["architectures"][0] ) + print("Model architecture not supported: " + hparams["architectures"][0]) sys.exit() model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True, trust_remote_code=True) @@ -68,18 +69,23 @@ def permute(weights: NDArray, n_head: int) -> NDArray: print("gguf: get model metadata") -llm_arch = "llama" -head_count = hparams["num_attention_heads"] +llm_arch = "llama" +hf_repo = hparams["_name_or_path"] +head_count = hparams["num_attention_heads"] +head_count_kv = hparams["num_key_value_heads"] block_count = hparams["num_hidden_layers"] gguf_writer.add_name(last_dir) gguf_writer.add_architecture(llm_arch) +gguf_writer.add_quantization_version(ftype) +guff_writer.add_source_hf_repo(hf_repo) gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) gguf_writer.add_block_count(llm_arch, block_count) gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"]) gguf_writer.add_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"]) gguf_writer.add_head_count(llm_arch, head_count) +gguf_writer.add_head_count_kv(llm_arch, head_count_kv) gguf_writer.add_layer_norm_rms_eps(llm_arch, hparams["rms_norm_eps"]) @@ -173,7 +179,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: # permute these if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): - data = permute(data,head_count) + data = permute(data, head_count) # map tensor names if name.endswith(".weight") and name[:-7] in tensor_map: @@ -181,11 +187,11 @@ def permute(weights: NDArray, n_head: int) -> NDArray: elif name.endswith(".bias") and name[:-5] in tensor_map: name = tensor_map[name[:-5]] + ".bias" else: - print( "Can not map tensor '" + name + "'" ) + print("Can not map tensor '" + name + "'") sys.exit() n_dims = len(data.shape) - data_dtype = data.dtype + data_dtype = data.dtype # print( name + " dims " + str(n_dims) + " dtype " + str(data.dtype) ) @@ -223,7 +229,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: data = permute(data, head_count) n_dims = len(data.shape) - data_dtype = data.dtype + data_dtype = data.dtype if data_dtype != np.float16 and data_dtype != np.float32: # convert any unsupported data types to float32 @@ -237,5 +243,5 @@ def permute(weights: NDArray, n_head: int) -> NDArray: gguf_writer.close() -print("gguf: model successfully exported to '" + fname_out + "'" ) +print("gguf: model successfully exported to '" + fname_out + "'") print("") diff --git a/gguf.py b/gguf.py index 75ba2870afc9a..de3e5bbfbe883 100644 --- a/gguf.py +++ b/gguf.py @@ -12,23 +12,10 @@ import numpy as np import sys + class GGMLQuantizationType(IntEnum): F32 = 0 F16 = 1 - Q4_0 = 2 - Q4_1 = 3 - # Q4_2 = 4 # support has been removed - # Q4_3 = 5 # support has been removed - Q5_0 = 6 - Q5_1 = 7 - Q8_0 = 8 - Q8_1 = 9 - Q2_K = 10 - Q3_K = 11 - Q4_K = 12 - Q5_K = 13 - Q6_K = 14 - Q8_K = 15 class GGUFValueType(IntEnum): @@ -143,7 +130,7 @@ def add_val(self: str, val: Any, vtype: GGUFValueType = None, add_vtype: bool = if add_vtype: self.kv_data += struct.pack(" Date: Sun, 13 Aug 2023 14:38:53 +0300 Subject: [PATCH 130/242] convert : rm quantization version --- convert-llama-h5-to-gguf.py | 85 +++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 53378e47c3863..574788ee0e29e 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -23,6 +23,7 @@ def permute(weights: NDArray, n_head: int) -> NDArray: .swapaxes(1, 2) .reshape(weights.shape)) + def count_model_parts(dir_model: str) -> int: num_parts = 0 for filename in os.listdir(dir_model): @@ -33,6 +34,7 @@ def count_model_parts(dir_model: str) -> int: print("gguf: found " + str(num_parts) + " model parts") return num_parts + if len(sys.argv) < 3: print("Usage: convert-h5-to-ggml.py dir-model ftype\n") print(" ftype == 0 -> float32") @@ -86,7 +88,6 @@ def count_model_parts(dir_model: str) -> int: gguf_writer.add_name(last_dir) gguf_writer.add_architecture(llm_arch) -gguf_writer.add_quantization_version(ftype) guff_writer.add_source_hf_repo(hf_repo) gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) @@ -187,7 +188,7 @@ def count_model_parts(dir_model: str) -> int: ) for part_name in part_names: - print("gguf: loading model part '"+ part_name + "'") + print("gguf: loading model part '" + part_name + "'") model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu") for name in model_part.keys(): @@ -205,7 +206,7 @@ def count_model_parts(dir_model: str) -> int: # permute these if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): - data = permute(data,head_count) + data = permute(data, head_count) # map tensor names if name.endswith(".weight") and name[:-7] in tensor_map: @@ -213,11 +214,11 @@ def count_model_parts(dir_model: str) -> int: elif name.endswith(".bias") and name[:-5] in tensor_map: name = tensor_map[name[:-5]] + ".bias" else: - print( "Can not map tensor '" + name + "'" ) + print("Can not map tensor '" + name + "'") sys.exit() n_dims = len(data.shape) - data_dtype = data.dtype + data_dtype = data.dtype # if f32 desired, convert any float16 to float32 if ftype == 0 and data.dtype == np.float16: @@ -254,60 +255,60 @@ def count_model_parts(dir_model: str) -> int: ) for part_name in part_names: - print("gguf: loading model part '"+ part_name + "'") + print("gguf: loading model part '" + part_name + "'") model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu") for name in model_part.keys(): data = model_part[name] -<<<<<<< HEAD - n_dims = len(data.shape) +<< << << < HEAD + n_dims = len(data.shape) data_dtype = data.dtype -======= - old_dtype = data.dtype +== == == = + old_dtype = data.dtype - # we don't need these - if name.endswith(".rotary_emb.inv_freq"): - continue ->>>>>>> 17800cd80fec468411481dc34a51d42a936442f1 + # we don't need these + if name.endswith(".rotary_emb.inv_freq"): + continue +>>>>>> > 17800cd80fec468411481dc34a51d42a936442f1 - # convert any unsupported data types to float32 - if data.dtype != torch.float16 and data.dtype != torch.float32: - data = data.to(torch.float32) + # convert any unsupported data types to float32 + if data.dtype != torch.float16 and data.dtype != torch.float32: + data = data.to(torch.float32) - data = data.squeeze().numpy() + data = data.squeeze().numpy() - # permute these - if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): - data = permute(data, head_count) + # permute these + if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): + data = permute(data, head_count) - # map tensor names - if name.endswith(".weight") and name[:-7] in tensor_map: - name = tensor_map[name[:-7]] + ".weight" - elif name.endswith(".bias") and name[:-5] in tensor_map: - name = tensor_map[name[:-5]] + ".bias" - else: - print( "Can not map tensor '" + name + "'" ) - sys.exit() + # map tensor names + if name.endswith(".weight") and name[:-7] in tensor_map: + name = tensor_map[name[:-7]] + ".weight" + elif name.endswith(".bias") and name[:-5] in tensor_map: + name = tensor_map[name[:-5]] + ".bias" + else: + print("Can not map tensor '" + name + "'" ) + sys.exit() - n_dims = len(data.shape) - data_dtype = data.dtype + n_dims = len(data.shape) + data_dtype = data.dtype - # if f32 desired, convert any float16 to float32 - if ftype == 0 and data.dtype == np.float16: - data = data.astype(np.float32) + # if f32 desired, convert any float16 to float32 + if ftype == 0 and data.dtype == np.float16: + data = data.astype(np.float32) - # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 - if ftype == 1 and data_dtype == np.float16 and n_dims == 1: - data = data.astype(np.float32) + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if ftype == 1 and data_dtype == np.float16 and n_dims == 1: + data = data.astype(np.float32) - # if f16 desired, convert any float32 2-dim weight tensors to float16 - if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: - data = data.astype(np.float16) + # if f16 desired, convert any float32 2-dim weight tensors to float16 + if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data = data.astype(np.float16) - print( name + ", shape " + str(len(data.shape)) + ", " + str(old_dtype) + " --> " + str(data.dtype)) + print(name + ", shape " + str(len(data.shape)) + ", " + str(old_dtype) + " --> " + str(data.dtype)) - gguf_writer.write_tensor_to_file(data) + gguf_writer.write_tensor_to_file(data) gguf_writer.close() From 2827b840e4235e02c32f93ca80af19cb0790d3c7 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 13 Aug 2023 13:54:10 +0200 Subject: [PATCH 131/242] convert-gptneox-h5-to-gguf.py : add file_type key --- convert-gptneox-h5-to-gguf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index a8a0c7e2df357..ba6e90e42ed40 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -90,8 +90,9 @@ def count_model_parts(dir_model: str) -> int: llm_arch = "gptneox" block_count = hparams["num_hidden_layers"] -gguf_writer.add_name(last_dir) gguf_writer.add_architecture(llm_arch) +gguf_writer.add_name(last_dir) +gguf_writer.add_file_type( "All tensors F32" if ftype == 0 else "Most tensors F16, some F32") gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) gguf_writer.add_block_count(llm_arch, block_count) From 6beebf3fd92aa2ab5c8b060b6d25c5e1d3d12e95 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Sun, 13 Aug 2023 14:11:01 +0200 Subject: [PATCH 132/242] gptneox-main.cpp : add file_type key --- gptneox-main.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gptneox-main.cpp b/gptneox-main.cpp index 4773dfd690461..63ee5e61c22ef 100644 --- a/gptneox-main.cpp +++ b/gptneox-main.cpp @@ -379,6 +379,8 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 if (keyidx != -1) { fprintf(stdout, "%s: model license = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } keyidx = gguf_find_key(ggufctx, "general.architecture"); if (keyidx != -1) { fprintf(stdout, "%s: model architecture = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + keyidx = gguf_find_key(ggufctx, "general.file_type"); + if (keyidx != -1) { fprintf(stdout, "%s: model file type = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } } // check required metadata From 24f48833ab5a1fcacb15c1908e13a80b35fa0f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Sun, 13 Aug 2023 16:55:42 +0300 Subject: [PATCH 133/242] fix conflicts --- convert-llama-h5-to-gguf.py | 82 +++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 574788ee0e29e..cf9f6f80246dc 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -23,7 +23,6 @@ def permute(weights: NDArray, n_head: int) -> NDArray: .swapaxes(1, 2) .reshape(weights.shape)) - def count_model_parts(dir_model: str) -> int: num_parts = 0 for filename in os.listdir(dir_model): @@ -34,7 +33,6 @@ def count_model_parts(dir_model: str) -> int: print("gguf: found " + str(num_parts) + " model parts") return num_parts - if len(sys.argv) < 3: print("Usage: convert-h5-to-ggml.py dir-model ftype\n") print(" ftype == 0 -> float32") @@ -188,7 +186,7 @@ def count_model_parts(dir_model: str) -> int: ) for part_name in part_names: - print("gguf: loading model part '" + part_name + "'") + print("gguf: loading model part '"+ part_name + "'") model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu") for name in model_part.keys(): @@ -206,7 +204,7 @@ def count_model_parts(dir_model: str) -> int: # permute these if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): - data = permute(data, head_count) + data = permute(data,head_count) # map tensor names if name.endswith(".weight") and name[:-7] in tensor_map: @@ -214,11 +212,11 @@ def count_model_parts(dir_model: str) -> int: elif name.endswith(".bias") and name[:-5] in tensor_map: name = tensor_map[name[:-5]] + ".bias" else: - print("Can not map tensor '" + name + "'") + print( "Can not map tensor '" + name + "'" ) sys.exit() n_dims = len(data.shape) - data_dtype = data.dtype + data_dtype = data.dtype # if f32 desired, convert any float16 to float32 if ftype == 0 and data.dtype == np.float16: @@ -255,60 +253,56 @@ def count_model_parts(dir_model: str) -> int: ) for part_name in part_names: - print("gguf: loading model part '" + part_name + "'") + print("gguf: loading model part '"+ part_name + "'") model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu") for name in model_part.keys(): data = model_part[name] -<< << << < HEAD - n_dims = len(data.shape) - data_dtype = data.dtype -== == == = - old_dtype = data.dtype + + old_dtype = data.dtype - # we don't need these - if name.endswith(".rotary_emb.inv_freq"): - continue ->>>>>> > 17800cd80fec468411481dc34a51d42a936442f1 + # we don't need these + if name.endswith(".rotary_emb.inv_freq"): + continue - # convert any unsupported data types to float32 - if data.dtype != torch.float16 and data.dtype != torch.float32: - data = data.to(torch.float32) + # convert any unsupported data types to float32 + if data.dtype != torch.float16 and data.dtype != torch.float32: + data = data.to(torch.float32) - data = data.squeeze().numpy() + data = data.squeeze().numpy() - # permute these - if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): - data = permute(data, head_count) + # permute these + if name.endswith(".q_proj.weight") or name.endswith(".k_proj.weight"): + data = permute(data, head_count) - # map tensor names - if name.endswith(".weight") and name[:-7] in tensor_map: - name = tensor_map[name[:-7]] + ".weight" - elif name.endswith(".bias") and name[:-5] in tensor_map: - name = tensor_map[name[:-5]] + ".bias" - else: - print("Can not map tensor '" + name + "'" ) - sys.exit() + # map tensor names + if name.endswith(".weight") and name[:-7] in tensor_map: + name = tensor_map[name[:-7]] + ".weight" + elif name.endswith(".bias") and name[:-5] in tensor_map: + name = tensor_map[name[:-5]] + ".bias" + else: + print( "Can not map tensor '" + name + "'" ) + sys.exit() - n_dims = len(data.shape) - data_dtype = data.dtype + n_dims = len(data.shape) + data_dtype = data.dtype - # if f32 desired, convert any float16 to float32 - if ftype == 0 and data.dtype == np.float16: - data = data.astype(np.float32) + # if f32 desired, convert any float16 to float32 + if ftype == 0 and data.dtype == np.float16: + data = data.astype(np.float32) - # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 - if ftype == 1 and data_dtype == np.float16 and n_dims == 1: - data = data.astype(np.float32) + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if ftype == 1 and data_dtype == np.float16 and n_dims == 1: + data = data.astype(np.float32) - # if f16 desired, convert any float32 2-dim weight tensors to float16 - if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: - data = data.astype(np.float16) + # if f16 desired, convert any float32 2-dim weight tensors to float16 + if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data = data.astype(np.float16) - print(name + ", shape " + str(len(data.shape)) + ", " + str(old_dtype) + " --> " + str(data.dtype)) + print( name + ", shape " + str(len(data.shape)) + ", " + str(old_dtype) + " --> " + str(data.dtype)) - gguf_writer.write_tensor_to_file(data) + gguf_writer.write_tensor_to_file(data) gguf_writer.close() From 196b50fee74625528fae91e7125dc1e9cc07cecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Yusuf=20Sar=C4=B1g=C3=B6z?= Date: Mon, 14 Aug 2023 08:50:47 +0300 Subject: [PATCH 134/242] gguf : add todos and comments --- gguf-llama.cpp | 64 ++++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 700d6009bcddb..826af549b5459 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -258,6 +258,11 @@ struct llama_kv_cache { }; struct llama_vocab { + // TODO: convert to this gguf_vocab + // add a vector of merges + // add members for bos/eos/pad/sep tokens + // so that we can pass it to different types of tokenizers with a common interface + using id = int32_t; using token = std::string; @@ -447,7 +452,7 @@ static size_t llama_calc_tensor_size(const std::vector & ne, enum ggml return size / ggml_blck_size(type); } -struct llama_load_tensor { +struct gguf_load_tensor { std::string name; enum ggml_type type = GGML_TYPE_F32; std::vector ne; @@ -457,9 +462,9 @@ struct llama_load_tensor { uint8_t * data; }; -struct llama_load_tensors_map { +struct gguf_load_tensors_map { // tensors is kept in a separate vector to preserve file order - std::vector tensors; + std::vector tensors; std::unordered_map name_to_idx; }; @@ -477,7 +482,7 @@ struct gguf_file_loader { llama_vocab vocab; struct ggml_context * ctx_data = NULL; - gguf_file_loader(const char * fname, llama_load_tensors_map & tensors_map) + gguf_file_loader(const char * fname, gguf_load_tensors_map & tensors_map) : file(fname, "rb") { fprintf(stderr, "llama.cpp: loading model from %s\n", fname); @@ -523,13 +528,9 @@ struct ggml_context * ctx_data = NULL; void read_hparams() { - // TODO make keysconstants in header + // TODO define keys as constants in header // TODO: read all hparams from file - int q_ver_idx = gguf_find_key (gguf_ctx, "general.quantization_version"); - if (q_ver_idx != -1) { - hparams.ftype = (enum llama_ftype) gguf_get_val_u32(gguf_ctx, q_ver_idx); - } - + hparams.n_vocab = read_n_vocab(); hparams.n_ctx = read_u32("llama.context_length"); hparams.n_embd = read_u32("llama.embedding_length"); @@ -567,11 +568,11 @@ struct ggml_context * ctx_data = NULL; } } - void read_tensor_metadata(llama_load_tensors_map & tensors_map) { + void read_tensor_metadata(gguf_load_tensors_map & tensors_map) { const int n_tensors = gguf_get_n_tensors(gguf_ctx); for (int i = 0; i < n_tensors; ++i) { - llama_load_tensor tensor; + gguf_load_tensor tensor; const char * name = gguf_get_tensor_name(gguf_ctx, i); struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); @@ -617,6 +618,13 @@ struct ggml_context * ctx_data = NULL; }; struct gguf_file_saver { + // TODO + // this implementation now assumes that the data section is of the same length as the unquantized model. + // this is needed to write tensor metadata and weights in a single pass by seeking to appropriate positions in the file. + // this may not be true when we add quantization version and change ftype description (currently it's string according to the specs, + // but better to have it as uint32). + // we need to calculate the delta in number of bytes written with a counter as a struct member. + gguf_file file; gguf_file_loader * fl; size_t info_offset; @@ -747,7 +755,7 @@ struct gguf_file_saver { GGML_ASSERT(info_offset == file.tell()); } - size_t write_tensor_info(llama_load_tensor & tensor, enum ggml_type type) { + size_t write_tensor_info(gguf_load_tensor & tensor, enum ggml_type type) { size_t total_written = 0; file.seek(info_offset, SEEK_SET); GGML_ASSERT(info_offset == file.tell()); @@ -761,14 +769,14 @@ struct gguf_file_saver { total_written += file.write_i32(type); total_written += file.write_u64(tensor_offset); - info_offset += total_written; + info_offset += total_written; // position to write info of the next tensor file.seek(0, SEEK_END); return total_written; } - void write_tensor(llama_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { + void write_tensor(gguf_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { switch (new_type) { case GGML_TYPE_F32: case GGML_TYPE_F16: @@ -791,13 +799,13 @@ struct gguf_file_saver { size_t padded_size = GGML_PAD(new_size, GGUF_DEFAULT_ALIGNMENT); // TODO: handle custom alignment size_t pad = padded_size - new_size; file.write_zeros(pad); - tensor_offset += padded_size; + tensor_offset += padded_size; // offset of the next tensor } }; struct llama_model_loader { std::unique_ptr file_loader; - llama_load_tensors_map tensors_map; + gguf_load_tensors_map tensors_map; bool use_mmap; size_t num_ggml_tensors_created = 0; struct ggml_context * ggml_ctx = NULL; @@ -813,7 +821,7 @@ struct llama_model_loader { void calc_sizes(size_t * ctx_size_p, size_t * mmapped_size_p) const { *ctx_size_p = *mmapped_size_p = 0; - for (const llama_load_tensor & lt : tensors_map.tensors) { + for (const gguf_load_tensor & lt : tensors_map.tensors) { *ctx_size_p += sizeof(struct ggml_tensor) + GGML_OBJECT_SIZE; *(use_mmap ? mmapped_size_p : ctx_size_p) += lt.size + 16; } @@ -824,7 +832,7 @@ struct llama_model_loader { if (it == tensors_map.name_to_idx.end()) { throw std::runtime_error(std::runtime_error(format("llama.cpp: tensor '%s' is missing from model", name.c_str()))); } - llama_load_tensor & lt = tensors_map.tensors.at(it->second); + gguf_load_tensor & lt = tensors_map.tensors.at(it->second); if (lt.ne != ne) { throw std::runtime_error(format("llama.cpp: tensor '%s' has wrong shape; expected %s, got %s", name.c_str(), llama_format_tensor_shape(ne).c_str(), llama_format_tensor_shape(lt.ne).c_str())); @@ -833,7 +841,7 @@ struct llama_model_loader { return get_tensor_for(lt, backend); } - struct ggml_tensor * get_tensor_for(llama_load_tensor & lt, ggml_backend backend) { + struct ggml_tensor * get_tensor_for(gguf_load_tensor & lt, ggml_backend backend) { struct ggml_tensor * tensor; if (backend != GGML_BACKEND_CPU) { ggml_set_no_alloc(ggml_ctx, true); @@ -866,7 +874,7 @@ struct llama_model_loader { size_t data_size = 0; size_t prefetch_size = 0; size_t lock_size = 0; - for (const llama_load_tensor & lt : tensors_map.tensors) { + for (const gguf_load_tensor & lt : tensors_map.tensors) { data_size += lt.size; if (lt.ggml_tensor->backend == GGML_BACKEND_CPU) { prefetch_size += lt.size; @@ -881,7 +889,7 @@ struct llama_model_loader { } size_t done_size = 0; - for (llama_load_tensor & lt : tensors_map.tensors) { + for (gguf_load_tensor & lt : tensors_map.tensors) { if (progress_callback) { progress_callback((float) done_size / data_size, progress_callback_user_data); } @@ -928,7 +936,7 @@ struct llama_model_loader { } } - void load_data_for(llama_load_tensor & lt) { + void load_data_for(gguf_load_tensor & lt) { if (use_mmap) { lt.data = (uint8_t *) mapping->addr + lt.file_off; } else { @@ -942,7 +950,7 @@ struct llama_model_loader { } } - static void print_checksum(llama_load_tensor & lt) { + static void print_checksum(gguf_load_tensor & lt) { uint32_t sum = 0; for (size_t i = 0; i < lt.size; i++) { uint8_t byte = lt.data[i]; @@ -1421,7 +1429,7 @@ static void llama_model_load_internal( } // populate `tensors_by_name` - for (llama_load_tensor & lt : ml->tensors_map.tensors) { + for (gguf_load_tensor & lt : ml->tensors_map.tensors) { model.tensors_by_name.emplace_back(lt.name, lt.ggml_tensor); } @@ -2896,7 +2904,7 @@ void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar // quantization // -static void llama_convert_tensor_internal(const llama_load_tensor & tensor, gguf_buffer & output, const int nelements, const int nthread) { +static void llama_convert_tensor_internal(const gguf_load_tensor & tensor, gguf_buffer & output, const int nelements, const int nthread) { if (output.size < nelements * sizeof(float)) { output.resize(nelements * sizeof(float)); } @@ -3018,7 +3026,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s }; size_t idx = 0; - for (llama_load_tensor & tensor : model_loader->tensors_map.tensors) { + for (gguf_load_tensor & tensor : model_loader->tensors_map.tensors) { gguf_buffer read_data; read_data.resize(tensor.size); tensor.data = read_data.addr; @@ -3570,7 +3578,7 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const return 1; } size_t idx = model_loader->tensors_map.name_to_idx[base_name]; - llama_load_tensor & lt = model_loader->tensors_map.tensors[idx]; + gguf_load_tensor & lt = model_loader->tensors_map.tensors[idx]; base_t = model_loader->get_tensor(base_name, { (uint32_t)dest_t->ne[0], (uint32_t)dest_t->ne[1] }, GGML_BACKEND_CPU); lt.data = (uint8_t *) lt.ggml_tensor->data; model_loader->load_data_for(lt); From 5d22a9db133d0b5737f8889d488070a55dbe7682 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:55:44 +0200 Subject: [PATCH 135/242] convert-gptneox-h5-to-gguf.py : tensor name map changes --- convert-gptneox-h5-to-gguf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert-gptneox-h5-to-gguf.py b/convert-gptneox-h5-to-gguf.py index ba6e90e42ed40..a3d78c9ef2aa0 100644 --- a/convert-gptneox-h5-to-gguf.py +++ b/convert-gptneox-h5-to-gguf.py @@ -1,7 +1,7 @@ # HF gptneox--> gguf conversion import gguf -import gguf_tensor_map as tmap +import gguf_namemap as tmap import os import sys import struct @@ -188,7 +188,7 @@ def count_model_parts(dir_model: str) -> int: # TENSORS -tensor_map = tmap.get_tensor_map(block_count) +tensor_map = tmap.get_tensor_namemap(block_count) # tensor info print("gguf: get tensor metadata") From 51939d7d1bd6b752d66faf6050a9ab2d80915ba3 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:56:59 +0200 Subject: [PATCH 136/242] Create gguf_namemap.py : tensor name map changes --- gguf_namemap.py | 95 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 gguf_namemap.py diff --git a/gguf_namemap.py b/gguf_namemap.py new file mode 100644 index 0000000000000..7546630ed26c8 --- /dev/null +++ b/gguf_namemap.py @@ -0,0 +1,95 @@ +# Recommended mapping of model tensor names for storage in gguf + +def get_tensor_namemap( n_blocks : int): + tensor_map = {} + # Token embeddings + mapped_to = "token_embd" + tensor_map["gpt_neox.embed_in"] = mapped_to # gptneox + tensor_map["transformer.wte"] = mapped_to # gpt2 mpt + tensor_map["transformer.word_embeddings"] = mapped_to # falcon + tensor_map["model.embed_tokens"] = mapped_to # llama-hf + tensor_map["tok_embeddings"] = mapped_to # llama-pth + # Position embeddings + mapped_to = "pos_embd" + tensor_map["transformer.wpe"] = mapped_to # gpt2 + # Output norm + mapped_to = "output_norm" + tensor_map["gpt_neox.final_layer_norm"] = mapped_to # gptneox + tensor_map["transformer.ln_f"] = mapped_to # gpt2 falcon + tensor_map["transformer.norm_f"] = mapped_to # mpt + tensor_map["model.norm"] = mapped_to # llama-hf + tensor_map["norm"] = mapped_to # llama-pth + # Output + mapped_to = "output" + tensor_map["embed_out"] = mapped_to # gptneox + tensor_map["lm_head"] = mapped_to # gpt2 mpt falcon llama-hf + tensor_map["output"] = mapped_to # llama-pth + # Attention and fee-forward layer blocks + for i in range(0,n_blocks): + # Attention norm + mapped_to = "blk."+str(i)+".attn_norm" + tensor_map["gpt_neox.layers."+str(i)+".input_layernorm"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".ln_1"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".norm_1"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".input_layernorm"] = mapped_to # falcon7b + tensor_map["transformer.h."+str(i)+".ln_attn"] = mapped_to # falcon40b + tensor_map["model.layers."+str(i)+".input_layernorm"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention_norm"] = mapped_to # llama-pth + # Attention norm 2 + mapped_to = "blk."+str(i)+".attn_norm_2" + tensor_map["transformer.h."+str(i)+".ln_mlp"] = mapped_to # falcon40b + # Attention query-key-value + mapped_to = "blk."+str(i)+".attn_qkv" + tensor_map["gpt_neox.layers."+str(i)+".attention.query_key_value"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".attn.c_attn"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".attn.Wqkv"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".self_attention.query_key_value"] = mapped_to # falcon + # Attention query + mapped_to = "blk."+str(i)+".attn_q" + tensor_map["model.layers."+str(i)+".self_attn.q_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention.wq"] = mapped_to # llama-pth + # Attention key + mapped_to = "blk."+str(i)+".attn_k" + tensor_map["model.layers."+str(i)+".self_attn.k_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention.wk"] = mapped_to # llama-pth + # Attention value + mapped_to = "blk."+str(i)+".attn_v" + tensor_map["model.layers."+str(i)+".self_attn.v_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention.wv"] = mapped_to # llama-pth + # Attention output + mapped_to = "blk."+str(i)+".attn_output" + tensor_map["gpt_neox.layers."+str(i)+".attention.dense"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".attn.c_proj"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".attn.out_proj"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".self_attention.dense"] = mapped_to # falcon + tensor_map["model.layers."+str(i)+".self_attn.o_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".attention.wo"] = mapped_to # llama-pth + # Feed-forward norm + mapped_to = "blk."+str(i)+".ffn_norm" + tensor_map["gpt_neox.layers."+str(i)+".post_attention_layernorm"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".ln_2"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".norm_2"] = mapped_to # mpt + tensor_map["model.layers."+str(i)+".post_attention_layernorm"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".ffn_norm"] = mapped_to # llama-pth + # Feed-forward up + mapped_to = "blk."+str(i)+".ffn_up" + tensor_map["gpt_neox.layers."+str(i)+".mlp.dense_h_to_4h"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".mlp.c_fc"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".ffn.up_proj"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".mlp.dense_h_to_4h"] = mapped_to # falcon + tensor_map["model.layers."+str(i)+".mlp.up_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".feed_forward.w3"] = mapped_to # llama-pth + # Feed-forward gate + mapped_to = "blk."+str(i)+".ffn_gate" + tensor_map["model.layers."+str(i)+".mlp.gate_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".feed_forward.w1"] = mapped_to # llama-pth + # Feed-forward down + mapped_to = "blk."+str(i)+".ffn_down" + tensor_map["gpt_neox.layers."+str(i)+".mlp.dense_4h_to_h"] = mapped_to # gptneox + tensor_map["transformer.h."+str(i)+".mlp.c_proj"] = mapped_to # gpt2 + tensor_map["transformer.blocks."+str(i)+".ffn.down_proj"] = mapped_to # mpt + tensor_map["transformer.h."+str(i)+".mlp.dense_4h_to_h"] = mapped_to # falcon + tensor_map["model.layers."+str(i)+".mlp.down_proj"] = mapped_to # llama-hf + tensor_map["layers."+str(i)+".feed_forward.w2"] = mapped_to # llama-pth + + return tensor_map From 806a15749d6ba4793c38295f1bd1128158c8e244 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:57:19 +0200 Subject: [PATCH 137/242] Delete gguf_tensor_map.py --- gguf_tensor_map.py | 96 ---------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 gguf_tensor_map.py diff --git a/gguf_tensor_map.py b/gguf_tensor_map.py deleted file mode 100644 index d73788bb46877..0000000000000 --- a/gguf_tensor_map.py +++ /dev/null @@ -1,96 +0,0 @@ -# Recommended mapping of model tensor names for storage in gguf - -def get_tensor_map( n_blocks : int): - tensor_map = {} - # Token embeddings - mapped_to = "transformer.token_embd" - tensor_map["gpt_neox.embed_in"] = mapped_to # gptneox - tensor_map["transformer.wte"] = mapped_to # gpt2 mpt - tensor_map["transformer.word_embeddings"] = mapped_to # falcon - tensor_map["model.embed_tokens"] = mapped_to # llama-hf - tensor_map["tok_embeddings"] = mapped_to # llama-pth - # Position embeddings - mapped_to = "transformer.pos_embd" - tensor_map["transformer.wpe"] = mapped_to # gpt2 - # Output norm - mapped_to = "transformer.output_norm" - tensor_map["gpt_neox.final_layer_norm"] = mapped_to # gptneox - tensor_map["transformer.ln_f"] = mapped_to # gpt2 falcon - tensor_map["transformer.norm_f"] = mapped_to # mpt - tensor_map["model.norm"] = mapped_to # llama-hf - tensor_map["norm"] = mapped_to # llama-pth - # Output - mapped_to = "transformer.output" - tensor_map["embed_out"] = mapped_to # gptneox - tensor_map["lm_head"] = mapped_to # gpt2 mpt falcon llama-hf - tensor_map["output"] = mapped_to # llama-pth - # Attention and fee-forward layer blocks - for i in range(0,n_blocks): - # Attention norm - mapped_to = "transformer.blocks."+str(i)+".attn_norm" - tensor_map["gpt_neox.layers."+str(i)+".input_layernorm"] = mapped_to # gptneox - tensor_map["transformer.h."+str(i)+".ln_1"] = mapped_to # gpt2 - tensor_map["transformer.blocks."+str(i)+".norm_1"] = mapped_to # mpt - tensor_map["transformer.h."+str(i)+".input_layernorm"] = mapped_to # falcon7b - tensor_map["transformer.h."+str(i)+".ln_attn"] = mapped_to # falcon40b - tensor_map["model.layers."+str(i)+".input_layernorm"] = mapped_to # llama-hf - tensor_map["layers."+str(i)+".attention_norm"] = mapped_to # llama-pth - # Attention norm 2 - mapped_to = "transformer.blocks."+str(i)+".attn_norm_2" - tensor_map["transformer.h."+str(i)+".ln_mlp"] = mapped_to # falcon40b - # Attention query-key-value - mapped_to = "transformer.blocks."+str(i)+".attn_qkv" - tensor_map["gpt_neox.layers."+str(i)+".attention.query_key_value"] = mapped_to # gptneox - tensor_map["transformer.h."+str(i)+".attn.c_attn"] = mapped_to # gpt2 - tensor_map["transformer.blocks."+str(i)+".attn.Wqkv"] = mapped_to # mpt - tensor_map["transformer.h."+str(i)+".self_attention.query_key_value"] = mapped_to # falcon - # Attention query - mapped_to = "transformer.blocks."+str(i)+".attn_q" - tensor_map["model.layers."+str(i)+".self_attn.q_proj"] = mapped_to # llama-hf - tensor_map["layers."+str(i)+".attention.wq"] = mapped_to # llama-pth - # Attention key - mapped_to = "transformer.blocks."+str(i)+".attn_k" - tensor_map["model.layers."+str(i)+".self_attn.k_proj"] = mapped_to # llama-hf - tensor_map["layers."+str(i)+".attention.wk"] = mapped_to # llama-pth - # Attention value - mapped_to = "transformer.blocks."+str(i)+".attn_v" - tensor_map["model.layers."+str(i)+".self_attn.v_proj"] = mapped_to # llama-hf - tensor_map["layers."+str(i)+".attention.wv"] = mapped_to # llama-pth - # Attention output - mapped_to = "transformer.blocks."+str(i)+".attn_output" - tensor_map["gpt_neox.layers."+str(i)+".attention.dense"] = mapped_to # gptneox - tensor_map["transformer.h."+str(i)+".attn.c_proj"] = mapped_to # gpt2 - tensor_map["transformer.blocks."+str(i)+".attn.out_proj"] = mapped_to # mpt - tensor_map["transformer.h."+str(i)+".self_attention.dense"] = mapped_to # falcon - tensor_map["model.layers."+str(i)+".self_attn.o_proj"] = mapped_to # llama-hf - tensor_map["layers."+str(i)+".attention.wo"] = mapped_to # llama-pth - # Feed-forward norm - mapped_to = "transformer.blocks."+str(i)+".ffn_norm" - tensor_map["gpt_neox.layers."+str(i)+".post_attention_layernorm"] = mapped_to # gptneox - tensor_map["transformer.h."+str(i)+".ln_2"] = mapped_to # gpt2 - tensor_map["transformer.blocks."+str(i)+".norm_2"] = mapped_to # mpt - tensor_map["model.layers."+str(i)+".post_attention_layernorm"] = mapped_to # llama-hf - tensor_map["layers."+str(i)+".ffn_norm"] = mapped_to # llama-pth - # Feed-forward up - mapped_to = "transformer.blocks."+str(i)+".ffn_up" - tensor_map["gpt_neox.layers."+str(i)+".mlp.dense_h_to_4h"] = mapped_to # gptneox - tensor_map["transformer.h."+str(i)+".mlp.c_fc"] = mapped_to # gpt2 - tensor_map["transformer.blocks."+str(i)+".ffn.up_proj"] = mapped_to # mpt - tensor_map["transformer.h."+str(i)+".mlp.dense_h_to_4h"] = mapped_to # falcon - tensor_map["model.layers."+str(i)+".mlp.up_proj"] = mapped_to # llama-hf - tensor_map["layers."+str(i)+".feed_forward.w3"] = mapped_to # llama-pth - # Feed-forward gate - mapped_to = "transformer.blocks."+str(i)+".ffn_gate" - tensor_map["model.layers."+str(i)+".mlp.gate_proj"] = mapped_to # llama-hf - tensor_map["layers."+str(i)+".feed_forward.w1"] = mapped_to # llama-pth - # Feed-forward down - mapped_to = "transformer.blocks."+str(i)+".ffn_down" - tensor_map["gpt_neox.layers."+str(i)+".mlp.dense_4h_to_h"] = mapped_to # gptneox - tensor_map["transformer.h."+str(i)+".mlp.c_proj"] = mapped_to # gpt2 - tensor_map["transformer.blocks."+str(i)+".ffn.down_proj"] = mapped_to # mpt - tensor_map["transformer.h."+str(i)+".mlp.dense_4h_to_h"] = mapped_to # falcon - tensor_map["model.layers."+str(i)+".mlp.down_proj"] = mapped_to # llama-hf - tensor_map["layers."+str(i)+".feed_forward.w2"] = mapped_to # llama-pth - - return tensor_map - From d753dfbcc89cc0c7fdc54ef2f34e1aeccbbe7170 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:59:18 +0200 Subject: [PATCH 138/242] gptneox-main.cpp : tensor name map changes --- gptneox-main.cpp | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/gptneox-main.cpp b/gptneox-main.cpp index 63ee5e61c22ef..f336ee88bfaf9 100644 --- a/gptneox-main.cpp +++ b/gptneox-main.cpp @@ -370,17 +370,19 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 int keyidx; keyidx = gguf_find_key(ggufctx, "general.name"); - if (keyidx != -1) { fprintf(stdout, "%s: model name = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + if (keyidx != -1) { fprintf(stdout, "%s: model name = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } keyidx = gguf_find_key(ggufctx, "general.description"); - if (keyidx != -1) { fprintf(stdout, "%s: model description = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + if (keyidx != -1) { fprintf(stdout, "%s: model description = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } keyidx = gguf_find_key(ggufctx, "general.author"); - if (keyidx != -1) { fprintf(stdout, "%s: model author = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + if (keyidx != -1) { fprintf(stdout, "%s: model author = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } keyidx = gguf_find_key(ggufctx, "general.license"); - if (keyidx != -1) { fprintf(stdout, "%s: model license = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + if (keyidx != -1) { fprintf(stdout, "%s: model license = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } keyidx = gguf_find_key(ggufctx, "general.architecture"); - if (keyidx != -1) { fprintf(stdout, "%s: model architecture = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + if (keyidx != -1) { fprintf(stdout, "%s: model architecture = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } keyidx = gguf_find_key(ggufctx, "general.file_type"); - if (keyidx != -1) { fprintf(stdout, "%s: model file type = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + if (keyidx != -1) { fprintf(stdout, "%s: model file type = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } + keyidx = gguf_find_key(ggufctx, "general.source.hugginface.repository"); + if (keyidx != -1) { fprintf(stdout, "%s: model source HF repo = %s\n", __func__, gguf_get_val_str(ggufctx, keyidx)); } } // check required metadata @@ -551,21 +553,21 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt2 model.blocks.resize(n_block); - model.wte = ggml_get_tensor(ctx, "transformer.token_embd.weight"); - model.ln_f_g = ggml_get_tensor(ctx, "transformer.output_norm.weight"); - model.ln_f_b = ggml_get_tensor(ctx, "transformer.output_norm.bias"); - model.lmh_g = ggml_get_tensor(ctx, "transformer.output.weight"); + model.wte = ggml_get_tensor(ctx, "token_embd.weight"); + model.ln_f_g = ggml_get_tensor(ctx, "output_norm.weight"); + model.ln_f_b = ggml_get_tensor(ctx, "output_norm.bias"); + model.lmh_g = ggml_get_tensor(ctx, "output.weight"); // map by name - model.tensors["transformer.token_embd.weight"] = model.wte; - model.tensors["transformer.output_norm.weight"] = model.ln_f_g; - model.tensors["transformer.output_norm.bias"] = model.ln_f_b; - model.tensors["transformer.output.weight"] = model.lmh_g; + model.tensors["token_embd.weight"] = model.wte; + model.tensors["output_norm.weight"] = model.ln_f_g; + model.tensors["output_norm.bias"] = model.ln_f_b; + model.tensors["output.weight"] = model.lmh_g; for (int i = 0; i < n_block; ++i) { auto & block = model.blocks[i]; - std::string blocknamestart = "transformer.blocks." + std::to_string(i) + "."; + std::string blocknamestart = "blk." + std::to_string(i) + "."; block.ln_1_g = get_tensor_ex(ctx, blocknamestart + "attn_norm.weight" ); block.ln_1_b = get_tensor_ex(ctx, blocknamestart + "attn_norm.bias" ); From a7d226f87186c6ceed2cbbc469dd24f86471e53a Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:14:24 +0200 Subject: [PATCH 139/242] convert-llama-h5-to-gguf.py : fixes --- convert-llama-h5-to-gguf.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index cf9f6f80246dc..402fdb68b1e77 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -1,7 +1,7 @@ # HF llama --> gguf conversion, GQA/70b not supported import gguf -import gguf_tensor_map as tmap +import gguf_namemap as tmap import os import sys import struct @@ -79,14 +79,23 @@ def count_model_parts(dir_model: str) -> int: print("gguf: get model metadata") llm_arch = "llama" -hf_repo = hparams["_name_or_path"] -head_count = hparams["num_attention_heads"] -head_count_kv = hparams["num_key_value_heads"] block_count = hparams["num_hidden_layers"] +head_count = hparams["num_attention_heads"] + +if "num_key_value_heads" in hparams: + head_count_kv = hparams["num_key_value_heads"] +else: + head_count_kv = head_count + +if "_name_or_path" in hparams: + hf_repo = hparams["_name_or_path"] +else: + hf_repo="" -gguf_writer.add_name(last_dir) gguf_writer.add_architecture(llm_arch) -guff_writer.add_source_hf_repo(hf_repo) +gguf_writer.add_name(last_dir) +gguf_writer.add_file_type( "All tensors F32" if ftype == 0 else "Most tensors F16, some F32") +gguf_writer.add_source_hf_repo(hf_repo) gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) gguf_writer.add_block_count(llm_arch, block_count) @@ -173,7 +182,7 @@ def count_model_parts(dir_model: str) -> int: # TENSORS -tensor_map = tmap.get_tensor_map(block_count) +tensor_map = tmap.get_tensor_namemap(block_count) # tensor info print("gguf: get tensor metadata") From 5c5a95ba2dab6cdece5c5640056e3ed35f14074b Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:22:06 +0200 Subject: [PATCH 140/242] gguf.py : dont add empty strings --- gguf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gguf.py b/gguf.py index de3e5bbfbe883..18f42267aa78d 100644 --- a/gguf.py +++ b/gguf.py @@ -114,6 +114,7 @@ def add_bool(self, key: str, val: bool): self.add_val(val, GGUFValueType.BOOL) def add_string(self, key: str, val: str): + if len(val) == 0: return self.add_key(key) self.add_val(val, GGUFValueType.STRING) From 0c19ae70d5a10e563b43a4cbd7112a2df74f2246 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 14 Aug 2023 12:56:48 +0300 Subject: [PATCH 141/242] simple : minor style changes --- convert-llama-h5-to-gguf.py | 10 +- examples/gguf/gguf-llama-simple.cpp | 138 ++++++++------------------- examples/simple/simple.cpp | 141 +++++++++------------------- gguf-util.h | 3 +- 4 files changed, 91 insertions(+), 201 deletions(-) diff --git a/convert-llama-h5-to-gguf.py b/convert-llama-h5-to-gguf.py index 402fdb68b1e77..0bce659e697df 100644 --- a/convert-llama-h5-to-gguf.py +++ b/convert-llama-h5-to-gguf.py @@ -2,17 +2,18 @@ import gguf import gguf_namemap as tmap + import os import sys import struct import json import numpy as np +import torch + from typing import Any, List from pathlib import Path -import torch from sentencepiece import SentencePieceProcessor - #NDArray = np.ndarray[Any, Any] # compatible with python < 3.9 NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]' @@ -225,7 +226,7 @@ def count_model_parts(dir_model: str) -> int: sys.exit() n_dims = len(data.shape) - data_dtype = data.dtype + data_dtype = data.dtype # if f32 desired, convert any float16 to float32 if ftype == 0 and data.dtype == np.float16: @@ -268,7 +269,6 @@ def count_model_parts(dir_model: str) -> int: for name in model_part.keys(): data = model_part[name] - old_dtype = data.dtype # we don't need these @@ -295,7 +295,7 @@ def count_model_parts(dir_model: str) -> int: sys.exit() n_dims = len(data.shape) - data_dtype = data.dtype + data_dtype = data.dtype # if f32 desired, convert any float16 to float32 if ftype == 0 and data.dtype == np.float16: diff --git a/examples/gguf/gguf-llama-simple.cpp b/examples/gguf/gguf-llama-simple.cpp index 35c3c818349be..fa8fc6fef9f41 100644 --- a/examples/gguf/gguf-llama-simple.cpp +++ b/examples/gguf/gguf-llama-simple.cpp @@ -6,177 +6,121 @@ #include "gguf-llama.h" #include "build-info.h" -#include -#include #include #include -#include -#include -#include -#include #include #include -#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) -#include -#include -#elif defined (_WIN32) -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include -#include -#endif - - - -int main(int argc, char ** argv) -{ +int main(int argc, char ** argv) { gpt_params params; - //--------------------------------- - // Print help : - //--------------------------------- - - if ( argc == 1 || argv[1][0] == '-' ) - { - printf( "usage: %s MODEL_PATH [PROMPT]\n" , argv[0] ); + if (argc == 1 || argv[1][0] == '-') { + printf("usage: %s MODEL_PATH [PROMPT]\n" , argv[0]); return 1 ; } - //--------------------------------- - // Load parameters : - //--------------------------------- - - if ( argc >= 2 ) - { + if (argc >= 2) { params.model = argv[1]; } - if ( argc >= 3 ) - { + if (argc >= 3) { params.prompt = argv[2]; } - if ( params.prompt.empty() ) - { + if (params.prompt.empty()) { params.prompt = "Hello my name is"; } - //--------------------------------- - // Init LLM : - //--------------------------------- + // init LLM llama_backend_init(params.numa); llama_context_params ctx_params = llama_context_default_params(); llama_model * model = llama_load_model_from_file(params.model.c_str(), ctx_params); - - if ( model == NULL ) - { - fprintf( stderr , "%s: error: unable to load model\n" , __func__ ); + + if (model == NULL) { + fprintf(stderr , "%s: error: unable to load model\n" , __func__); return 1; } llama_context * ctx = llama_new_context_with_model(model, ctx_params); - //--------------------------------- - // Tokenize the prompt : - //--------------------------------- + // tokenize the prompt std::vector tokens_list; - tokens_list = ::llama_tokenize( ctx , params.prompt , true ); + tokens_list = ::llama_tokenize(ctx, params.prompt, true); - const int max_context_size = llama_n_ctx( ctx ); - const int max_tokens_list_size = max_context_size - 4 ; + const int max_context_size = llama_n_ctx(ctx); + const int max_tokens_list_size = max_context_size - 4; - if ( (int)tokens_list.size() > max_tokens_list_size ) - { - fprintf( stderr , "%s: error: prompt too long (%d tokens, max %d)\n" , - __func__ , (int)tokens_list.size() , max_tokens_list_size ); + if ((int)tokens_list.size() > max_tokens_list_size) { + fprintf(stderr, "%s: error: prompt too long (%d tokens, max %d)\n", __func__, (int) tokens_list.size(), max_tokens_list_size); return 1; } - fprintf( stderr, "\n\n" ); - - // Print the tokens from the prompt : + fprintf(stderr, "\n\n"); - for( auto id : tokens_list ) - { - printf( "%s" , llama_token_to_str( ctx , id ) ); + for (auto id : tokens_list) { + fprintf(stderr, "%s", llama_token_to_str(ctx, id)); } - fflush(stdout); - + fflush(stderr); - //--------------------------------- - // Main prediction loop : - //--------------------------------- + // main loop // The LLM keeps a contextual cache memory of previous token evaluation. // Usually, once this cache is full, it is required to recompute a compressed context based on previous // tokens (see "infinite text generation via context swapping" in the main example), but in this minimalist // example, we will just stop the loop once this cache is full or once an end of stream is detected. - while ( llama_get_kv_cache_token_count( ctx ) < max_context_size ) - { - //--------------------------------- - // Evaluate the tokens : - //--------------------------------- + while (llama_get_kv_cache_token_count(ctx) < max_context_size) { + // evaluate the transformer - if ( llama_eval( ctx , tokens_list.data() , int(tokens_list.size()) , llama_get_kv_cache_token_count( ctx ) , params.n_threads ) ) - { - fprintf( stderr, "%s : failed to eval\n" , __func__ ); + if (llama_eval(ctx, tokens_list.data(), int(tokens_list.size()), llama_get_kv_cache_token_count(ctx), params.n_threads)) { + fprintf(stderr, "%s : failed to eval\n", __func__); return 1; } tokens_list.clear(); - //--------------------------------- - // Select the best prediction : - //--------------------------------- + // sample the next token llama_token new_token_id = 0; - auto logits = llama_get_logits( ctx ); - auto n_vocab = llama_n_vocab( ctx ); // the size of the LLM vocabulary (in tokens) + auto logits = llama_get_logits(ctx); + auto n_vocab = llama_n_vocab(ctx); std::vector candidates; - candidates.reserve( n_vocab ); + candidates.reserve(n_vocab); - for( llama_token token_id = 0 ; token_id < n_vocab ; token_id++ ) - { - candidates.emplace_back( llama_token_data{ token_id , logits[ token_id ] , 0.0f } ); + for (llama_token token_id = 0; token_id < n_vocab; token_id++) { + candidates.emplace_back(llama_token_data{ token_id, logits[token_id], 0.0f }); } llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; - // Select it using the "Greedy sampling" method : - new_token_id = llama_sample_token_greedy( ctx , &candidates_p ); - + new_token_id = llama_sample_token_greedy(ctx , &candidates_p); // is it an end of stream ? - if ( new_token_id == llama_token_eos() ) - { + if (new_token_id == llama_token_eos()) { fprintf(stderr, " [end of text]\n"); break; } - // Print the new token : - printf( "%s" , llama_token_to_str( ctx , new_token_id ) ); - fflush( stdout ); + // print the new token : + printf("%s", llama_token_to_str(ctx, new_token_id)); + fflush(stdout); - // Push this new token for next evaluation : - tokens_list.push_back( new_token_id ); + // push this new token for next evaluation + tokens_list.push_back(new_token_id); - } // wend of main loop + } - llama_free( ctx ); - llama_free_model( model ); + llama_free(ctx); + llama_free_model(model); llama_backend_free(); return 0; } - -// EOF diff --git a/examples/simple/simple.cpp b/examples/simple/simple.cpp index 97137a6584aa3..55cce10442d58 100644 --- a/examples/simple/simple.cpp +++ b/examples/simple/simple.cpp @@ -2,180 +2,125 @@ #define _GNU_SOURCE #endif +#include "build-info.h" + #include "common.h" #include "llama.h" -#include "build-info.h" -#include -#include #include #include -#include -#include -#include -#include #include #include -#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) -#include -#include -#elif defined (_WIN32) -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include -#include -#endif - - - -int main(int argc, char ** argv) -{ +int main(int argc, char ** argv) { gpt_params params; - //--------------------------------- - // Print help : - //--------------------------------- - - if ( argc == 1 || argv[1][0] == '-' ) - { - printf( "usage: %s MODEL_PATH [PROMPT]\n" , argv[0] ); + if (argc == 1 || argv[1][0] == '-') { + printf("usage: %s MODEL_PATH [PROMPT]\n" , argv[0]); return 1 ; } - //--------------------------------- - // Load parameters : - //--------------------------------- - - if ( argc >= 2 ) - { + if (argc >= 2) { params.model = argv[1]; } - if ( argc >= 3 ) - { + if (argc >= 3) { params.prompt = argv[2]; } - if ( params.prompt.empty() ) - { + if (params.prompt.empty()) { params.prompt = "Hello my name is"; } - //--------------------------------- - // Init LLM : - //--------------------------------- + // init LLM llama_backend_init(params.numa); llama_model * model; llama_context * ctx; - std::tie(model, ctx) = llama_init_from_gpt_params( params ); + std::tie(model, ctx) = llama_init_from_gpt_params(params); - if ( model == NULL ) - { - fprintf( stderr , "%s: error: unable to load model\n" , __func__ ); + if (model == NULL) { + fprintf(stderr, "%s: error: unable to load model\n", __func__); return 1; } - //--------------------------------- - // Tokenize the prompt : - //--------------------------------- + // tokenize the prompt std::vector tokens_list; - tokens_list = ::llama_tokenize( ctx , params.prompt , true ); + tokens_list = ::llama_tokenize(ctx, params.prompt, true); - const int max_context_size = llama_n_ctx( ctx ); - const int max_tokens_list_size = max_context_size - 4 ; + const int max_context_size = llama_n_ctx(ctx); + const int max_tokens_list_size = max_context_size - 4; - if ( (int)tokens_list.size() > max_tokens_list_size ) - { - fprintf( stderr , "%s: error: prompt too long (%d tokens, max %d)\n" , - __func__ , (int)tokens_list.size() , max_tokens_list_size ); + if ((int)tokens_list.size() > max_tokens_list_size) { + fprintf(stderr, "%s: error: prompt too long (%d tokens, max %d)\n", __func__, (int) tokens_list.size(), max_tokens_list_size); return 1; } - fprintf( stderr, "\n\n" ); + fprintf(stderr, "\n\n"); - // Print the tokens from the prompt : - - for( auto id : tokens_list ) - { - printf( "%s" , llama_token_to_str( ctx , id ) ); + for (auto id : tokens_list) { + fprintf(stderr, "%s", llama_token_to_str(ctx, id)); } - fflush(stdout); - + fflush(stderr); - //--------------------------------- - // Main prediction loop : - //--------------------------------- + // main loop // The LLM keeps a contextual cache memory of previous token evaluation. // Usually, once this cache is full, it is required to recompute a compressed context based on previous // tokens (see "infinite text generation via context swapping" in the main example), but in this minimalist // example, we will just stop the loop once this cache is full or once an end of stream is detected. - while ( llama_get_kv_cache_token_count( ctx ) < max_context_size ) - { - //--------------------------------- - // Evaluate the tokens : - //--------------------------------- + while (llama_get_kv_cache_token_count( ctx ) < max_context_size) { + // evaluate the transformer - if ( llama_eval( ctx , tokens_list.data() , int(tokens_list.size()) , llama_get_kv_cache_token_count( ctx ) , params.n_threads ) ) - { - fprintf( stderr, "%s : failed to eval\n" , __func__ ); + if (llama_eval(ctx, tokens_list.data(), int(tokens_list.size()), llama_get_kv_cache_token_count(ctx), params.n_threads)) { + fprintf(stderr, "%s : failed to eval\n", __func__); return 1; } tokens_list.clear(); - //--------------------------------- - // Select the best prediction : - //--------------------------------- + // sample the next token llama_token new_token_id = 0; - auto logits = llama_get_logits( ctx ); - auto n_vocab = llama_n_vocab( ctx ); // the size of the LLM vocabulary (in tokens) + auto logits = llama_get_logits(ctx); + auto n_vocab = llama_n_vocab(ctx); std::vector candidates; - candidates.reserve( n_vocab ); + candidates.reserve(n_vocab); - for( llama_token token_id = 0 ; token_id < n_vocab ; token_id++ ) - { - candidates.emplace_back( llama_token_data{ token_id , logits[ token_id ] , 0.0f } ); + for (llama_token token_id = 0; token_id < n_vocab; token_id++) { + candidates.emplace_back(llama_token_data{ token_id, logits[token_id], 0.0f }); } llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; - // Select it using the "Greedy sampling" method : - new_token_id = llama_sample_token_greedy( ctx , &candidates_p ); - + new_token_id = llama_sample_token_greedy(ctx , &candidates_p); // is it an end of stream ? - if ( new_token_id == llama_token_eos() ) - { + if (new_token_id == llama_token_eos()) { fprintf(stderr, " [end of text]\n"); break; } - // Print the new token : - printf( "%s" , llama_token_to_str( ctx , new_token_id ) ); - fflush( stdout ); + // print the new token : + printf("%s", llama_token_to_str(ctx, new_token_id)); + fflush(stdout); - // Push this new token for next evaluation : - tokens_list.push_back( new_token_id ); + // push this new token for next evaluation + tokens_list.push_back(new_token_id); - } // wend of main loop + } - llama_free( ctx ); - llama_free_model( model ); + llama_free(ctx); + llama_free_model(model); llama_backend_free(); return 0; } - -// EOF diff --git a/gguf-util.h b/gguf-util.h index db0233cfca098..6bbabf667911e 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -5,7 +5,9 @@ #ifndef GGUF_UTIL_H #define GGUF_UTIL_H + #include "ggml.h" + #include #include #include @@ -62,7 +64,6 @@ static std::string format(const char * fmt, ...) { return std::string(buf.data(), size); } - template static std::string to_string(const T & val) { std::stringstream ss; From 62490f1380a9f4b5036b7122b01449781233e5fa Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 14 Aug 2023 13:04:35 +0300 Subject: [PATCH 142/242] gguf : use UNIX line ending --- constants.py | 108 +- gguf-llama.cpp | 8394 ++++++++++++++++++++++++------------------------ gguf-llama.h | 898 +++--- gguf-util.h | 1134 +++---- gguf.py | 678 ++-- 5 files changed, 5606 insertions(+), 5606 deletions(-) diff --git a/constants.py b/constants.py index 7fa238a73278c..87d6b7a0687aa 100644 --- a/constants.py +++ b/constants.py @@ -1,54 +1,54 @@ -GGUF_MAGIC = 0x47475546 -GGUF_VERSION = 1 -GGUF_DEFAULT_ALIGNMENT = 32 - -# general -KEY_GENERAL_ARCHITECTURE = "general.architecture" -KEY_GENERAL_QUANTIZATION_VERSION = "general.quantization_version" -KEY_GENERAL_ALIGNMENT = "general.alignment" -KEY_GENERAL_NAME = "general.name" -KEY_GENERAL_AUTHOR = "general.author" -KEY_GENERAL_URL = "general.url" -KEY_GENERAL_DESCRIPTION = "general.description" -KEY_GENERAL_FILE_TYPE = "general.file_type" -KEY_GENERAL_LICENSE = "general.license" -KEY_GENERAL_SOURCE_URL = "general.source.url" -KEY_GENERAL_SOURCE_HF_REPO = "general.source.hugginface.repository" - -# LLM -KEY_LLM_CONTEXT_LENGTH = "{llm}.context_length" -KEY_LLM_EMBEDDING_LENGTH = "{llm}.embedding_length" -KEY_LLM_BLOCK_COUNT = "{llm}.block_count" -KEY_LLM_FEED_FORWARD_LENGTH = "{llm}.feed_forward_length" -KEY_LLM_USE_PARALLEL_RESIDUAL = "{llm}.use_parallel_residual" -KEY_LLM_TENSOR_DATA_LAYOUT = "{llm}.tensor_data_layout" - -# attention -KEY_ATTENTION_HEAD_COUNT = "{llm}.attention.head_count" -KEY_ATTENTION_HEAD_COUNT_KV = "{llm}.attention.head_count_kv" -KEY_ATTENTION_MAX_ALIBI_BIAS = "{llm}.attention.max_alibi_bias" -KEY_ATTENTION_CLAMP_KQV = "{llm}.attention.clamp_kqv" -KEY_ATTENTION_LAYERNORM_EPS = "{llm}.attention.layer_norm_epsilon" -KEY_ATTENTION_LAYERNORM_RMS_EPS = "{llm}.attention.layer_norm_rms_epsilon" - -# RoPE -KEY_ROPE_DIMENSION_COUNT = "{llm}.rope.dimension_count" -KEY_ROPE_SCALE = "{llm}.rope.scale" - -# tokenization -KEY_TOKENIZER_MODEL = "tokenizer.ggml.model" -KEY_TOKENIZER_LIST = "tokenizer.ggml.tokens" -KEY_TOKENIZER_SCORES = "tokenizer.ggml.scores" -KEY_TOKENIZER_MERGES = "tokenizer.ggml.merges" -KEY_TOKENIZER_BOS_ID = "tokenizer.ggml.bos_token_id" -KEY_TOKENIZER_EOS_ID = "tokenizer.ggml.eos_token_id" -KEY_TOKENIZER_UNK_ID = "tokenizer.ggml.unknown_token_id" -KEY_TOKENIZER_SEP_ID = "tokenizer.ggml.seperator_token_id" -KEY_TOKENIZER_PAD_ID = "tokenizer.ggml.padding_token_id" -KEY_TOKENIZER_HF_JSON = "tokenizer.huggingface.json" -KEY_TOKENIZER_RWKV = "tokenizer.rwkv.world" -KEY_TOKENIZER_BOS_ID = "tokenizer.ggml.bos_token_id" -KEY_TOKENIZER_EOS_ID = "tokenizer.ggml.eos_token_id" -KEY_TOKENIZER_UNK_ID = "tokenizer.ggml.unknown_token_id" -KEY_TOKENIZER_SEP_ID = "tokenizer.ggml.separator_token_id" -KEY_TOKENIZER_PAD_ID = "tokenizer.ggml.padding_token_id" +GGUF_MAGIC = 0x47475546 +GGUF_VERSION = 1 +GGUF_DEFAULT_ALIGNMENT = 32 + +# general +KEY_GENERAL_ARCHITECTURE = "general.architecture" +KEY_GENERAL_QUANTIZATION_VERSION = "general.quantization_version" +KEY_GENERAL_ALIGNMENT = "general.alignment" +KEY_GENERAL_NAME = "general.name" +KEY_GENERAL_AUTHOR = "general.author" +KEY_GENERAL_URL = "general.url" +KEY_GENERAL_DESCRIPTION = "general.description" +KEY_GENERAL_FILE_TYPE = "general.file_type" +KEY_GENERAL_LICENSE = "general.license" +KEY_GENERAL_SOURCE_URL = "general.source.url" +KEY_GENERAL_SOURCE_HF_REPO = "general.source.hugginface.repository" + +# LLM +KEY_LLM_CONTEXT_LENGTH = "{llm}.context_length" +KEY_LLM_EMBEDDING_LENGTH = "{llm}.embedding_length" +KEY_LLM_BLOCK_COUNT = "{llm}.block_count" +KEY_LLM_FEED_FORWARD_LENGTH = "{llm}.feed_forward_length" +KEY_LLM_USE_PARALLEL_RESIDUAL = "{llm}.use_parallel_residual" +KEY_LLM_TENSOR_DATA_LAYOUT = "{llm}.tensor_data_layout" + +# attention +KEY_ATTENTION_HEAD_COUNT = "{llm}.attention.head_count" +KEY_ATTENTION_HEAD_COUNT_KV = "{llm}.attention.head_count_kv" +KEY_ATTENTION_MAX_ALIBI_BIAS = "{llm}.attention.max_alibi_bias" +KEY_ATTENTION_CLAMP_KQV = "{llm}.attention.clamp_kqv" +KEY_ATTENTION_LAYERNORM_EPS = "{llm}.attention.layer_norm_epsilon" +KEY_ATTENTION_LAYERNORM_RMS_EPS = "{llm}.attention.layer_norm_rms_epsilon" + +# RoPE +KEY_ROPE_DIMENSION_COUNT = "{llm}.rope.dimension_count" +KEY_ROPE_SCALE = "{llm}.rope.scale" + +# tokenization +KEY_TOKENIZER_MODEL = "tokenizer.ggml.model" +KEY_TOKENIZER_LIST = "tokenizer.ggml.tokens" +KEY_TOKENIZER_SCORES = "tokenizer.ggml.scores" +KEY_TOKENIZER_MERGES = "tokenizer.ggml.merges" +KEY_TOKENIZER_BOS_ID = "tokenizer.ggml.bos_token_id" +KEY_TOKENIZER_EOS_ID = "tokenizer.ggml.eos_token_id" +KEY_TOKENIZER_UNK_ID = "tokenizer.ggml.unknown_token_id" +KEY_TOKENIZER_SEP_ID = "tokenizer.ggml.seperator_token_id" +KEY_TOKENIZER_PAD_ID = "tokenizer.ggml.padding_token_id" +KEY_TOKENIZER_HF_JSON = "tokenizer.huggingface.json" +KEY_TOKENIZER_RWKV = "tokenizer.rwkv.world" +KEY_TOKENIZER_BOS_ID = "tokenizer.ggml.bos_token_id" +KEY_TOKENIZER_EOS_ID = "tokenizer.ggml.eos_token_id" +KEY_TOKENIZER_UNK_ID = "tokenizer.ggml.unknown_token_id" +KEY_TOKENIZER_SEP_ID = "tokenizer.ggml.separator_token_id" +KEY_TOKENIZER_PAD_ID = "tokenizer.ggml.padding_token_id" diff --git a/gguf-llama.cpp b/gguf-llama.cpp index dd6c59df3503d..99da3c56dc92c 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -1,4197 +1,4197 @@ -// Defines fileno on msys: -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#include -#include -#include -#endif - -#include "gguf-util.h" -#include "gguf-llama.h" - -#include "ggml.h" -#ifdef GGML_USE_CUBLAS -#include "ggml-cuda.h" -#elif defined(GGML_USE_CLBLAST) -#include "ggml-opencl.h" -#endif - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif -#ifdef GGML_USE_MPI -#include "ggml-mpi.h" -#endif -#ifdef GGML_USE_K_QUANTS -#ifndef QK_K -#ifdef GGML_QKK_64 -#define QK_K 64 -#else -#define QK_K 256 -#endif -#endif -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(_MSC_VER) -#pragma warning(disable: 4244 4267) // possible loss of data -#endif - -#define LLAMA_USE_SCRATCH -#define LLAMA_MAX_SCRATCH_BUFFERS 16 - -// available llama models -enum e_model { - MODEL_UNKNOWN, - MODEL_3B, - MODEL_7B, - MODEL_13B, - MODEL_30B, - MODEL_65B, - MODEL_70B, -}; - -static const size_t kB = 1024; -static const size_t MB = 1024*1024; - -// computed for n_ctx == 2048 -// TODO: dynamically determine these sizes -// needs modifications in ggml - -typedef void (*offload_func_t)(struct ggml_tensor * tensor); - -void llama_nop(struct ggml_tensor * tensor) { // don't offload by default - (void) tensor; -} - -// -// ggml helpers -// - -static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) { - struct ggml_cplan plan = ggml_graph_plan(graph, n_threads); - - if (plan.work_size > 0) { - buf.resize(plan.work_size); - plan.work_data = buf.data(); - } - - ggml_graph_compute(graph, &plan); -} - -// -// memory sizes (calculated for n_batch == 512) -// - -static const std::map & MEM_REQ_SCRATCH0(int n_ctx) -{ - static std::map k_sizes = { - { MODEL_3B, ((size_t) n_ctx / 16ull + 92ull) * MB }, - { MODEL_7B, ((size_t) n_ctx / 16ull + 100ull) * MB }, - { MODEL_13B, ((size_t) n_ctx / 12ull + 120ull) * MB }, - { MODEL_30B, ((size_t) n_ctx / 9ull + 160ull) * MB }, - { MODEL_65B, ((size_t) n_ctx / 6ull + 256ull) * MB }, // guess - { MODEL_70B, ((size_t) n_ctx / 7ull + 164ull) * MB }, - }; - return k_sizes; -} - -static const std::map & MEM_REQ_SCRATCH1() -{ - static std::map k_sizes = { - { MODEL_3B, 128ull * MB }, - { MODEL_7B, 160ull * MB }, - { MODEL_13B, 192ull * MB }, - { MODEL_30B, 256ull * MB }, - { MODEL_65B, 384ull * MB }, // guess - { MODEL_70B, 304ull * MB }, - }; - return k_sizes; -} - -// used to store the compute graph tensors + non-scratch data -static const std::map & MEM_REQ_EVAL() -{ - static std::map k_sizes = { - { MODEL_3B, 8ull * MB }, - { MODEL_7B, 10ull * MB }, - { MODEL_13B, 12ull * MB }, - { MODEL_30B, 16ull * MB }, - { MODEL_65B, 24ull * MB }, // guess - { MODEL_70B, 24ull * MB }, - }; - return k_sizes; -} - -// amount of VRAM needed per batch size to hold temporary results -// the values for 3b and 65b are not derived from testing but instead chosen conservatively -static const std::map & VRAM_REQ_SCRATCH_BASE() -{ - static std::map k_sizes = { - { MODEL_3B, 512ull * kB }, - { MODEL_7B, 512ull * kB }, - { MODEL_13B, 640ull * kB }, - { MODEL_30B, 768ull * kB }, - { MODEL_65B, 1536ull * kB }, - { MODEL_70B, 1536ull * kB }, // TODO (likely can be reduced) - }; - return k_sizes; -} - -// amount of VRAM needed per batch size and context to hold temporary results -// the values for 3b and 65b are not derived from testing but instead chosen conservatively -static const std::map & VRAM_REQ_SCRATCH_PER_CONTEXT() -{ - static std::map k_sizes = { - { MODEL_3B, 128ull }, - { MODEL_7B, 128ull }, - { MODEL_13B, 160ull }, - { MODEL_30B, 208ull }, - { MODEL_65B, 416ull }, - { MODEL_70B, 416ull }, // TODO (likely can be reduced) - }; - return k_sizes; -} - -// default hparams (LLaMA 7B) -struct llama_hparams { - uint32_t n_vocab = 32000; - uint32_t n_ctx = 512; // this is provided as user input? - uint32_t n_embd = 4096; - uint32_t n_head = 32; - uint32_t n_head_kv = 32; - uint32_t n_layer = 32; - uint32_t n_rot = 64; - uint32_t n_ff = 11008; - - float f_rms_norm_eps = LLAMA_DEFAULT_RMS_EPS; - - float rope_freq_base = 10000.0f; - float rope_freq_scale = 1.0f; - - enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16; - - bool operator!=(const llama_hparams & other) const { - return static_cast(memcmp(this, &other, sizeof(llama_hparams))); // NOLINT - } - - uint32_t n_gqa() const { - return n_head/n_head_kv; - } - - uint32_t n_embd_head() const { - return n_embd/n_head; - } - - uint32_t n_embd_gqa() const { - return n_embd/n_gqa(); - } - - size_t kv_size() const { - size_t result = 2ull; - result *= (size_t) n_embd_gqa(); - result *= (size_t) n_ctx; - result *= (size_t) n_layer; - result *= sizeof(ggml_fp16_t); - return result; - } -}; - -struct llama_layer { - // normalization - struct ggml_tensor * attention_norm; - - // attention - struct ggml_tensor * wq; - struct ggml_tensor * wk; - struct ggml_tensor * wv; - struct ggml_tensor * wo; - - // normalization - struct ggml_tensor * ffn_norm; - - // ff - struct ggml_tensor * w1; - struct ggml_tensor * w2; - struct ggml_tensor * w3; -}; - -struct llama_kv_cache { - struct ggml_tensor * k = NULL; - struct ggml_tensor * v = NULL; - - struct ggml_context * ctx = NULL; - - gguf_ctx_buffer buf; - - int n; // number of tokens currently in the cache - - ~llama_kv_cache() { - if (ctx) { - ggml_free(ctx); - } - -#ifdef GGML_USE_CUBLAS - ggml_cuda_free_data(k); - ggml_cuda_free_data(v); -#endif // GGML_USE_CUBLAS - } -}; - -struct llama_vocab { - // TODO: convert to this gguf_vocab - // add a vector of merges - // add members for bos/eos/pad/sep tokens - // so that we can pass it to different types of tokenizers with a common interface - - using id = int32_t; - using token = std::string; - - struct token_score { - token tok; - float score; - }; - - std::unordered_map token_to_id; - std::vector id_to_token; -}; - -struct llama_model { - e_model type = MODEL_UNKNOWN; - - llama_hparams hparams; - - struct ggml_tensor * tok_embeddings; - - struct ggml_tensor * norm; - struct ggml_tensor * output; - - std::vector layers; - int n_gpu_layers; - - // context - struct ggml_context * ctx = NULL; - - // the model memory buffer - gguf_ctx_buffer buf; - - // model memory mapped file - std::unique_ptr mapping; - - // objects representing data potentially being locked in memory - gguf_mlock mlock_buf; - gguf_mlock mlock_mmap; - - // for quantize-stats only - std::vector> tensors_by_name; - - int64_t t_load_us = 0; - int64_t t_start_us = 0; - - llama_vocab vocab; - - ~llama_model() { - if (ctx) { - ggml_free(ctx); - } - -#ifdef GGML_USE_CUBLAS - for (size_t i = 0; i < tensors_by_name.size(); ++i) { - ggml_cuda_free_data(tensors_by_name[i].second); - } - ggml_cuda_free_scratch(); -#elif defined(GGML_USE_CLBLAST) - for (size_t i = 0; i < tensors_by_name.size(); ++i) { - ggml_cl_free_data(tensors_by_name[i].second); - } -#endif - } -}; - -struct llama_context { - llama_context(const llama_model & model) : model(model), t_load_us(model.t_load_us), t_start_us(model.t_start_us) {} -#ifdef GGML_USE_METAL - ~llama_context() { - if (ctx_metal) { - ggml_metal_free(ctx_metal); - } - } -#endif - std::mt19937 rng; - - bool has_evaluated_once = false; - - int64_t t_sample_us = 0; - int64_t t_eval_us = 0; - int64_t t_p_eval_us = 0; - - int32_t n_sample = 0; // number of tokens sampled - int32_t n_eval = 0; // number of eval calls - int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1) - - const llama_model & model; - - bool model_owner = false; - - int64_t t_load_us; - int64_t t_start_us; - - // key + value cache for the self attention - struct llama_kv_cache kv_self; - - size_t mem_per_token = 0; - - // decode output (2-dimensional array: [n_tokens][n_vocab]) - std::vector logits; - bool logits_all = false; - - // input embedding (1-dimensional array: [n_embd]) - std::vector embedding; - - // reusable buffer for `struct ggml_graph_plan.work_data` - std::vector work_buffer; - - // memory buffers used to evaluate the model - // TODO: move in llama_state - gguf_ctx_buffer buf_compute; - gguf_ctx_buffer buf_scratch[LLAMA_MAX_SCRATCH_BUFFERS]; - -#ifdef GGML_USE_METAL - ggml_metal_context * ctx_metal = NULL; -#endif - -#ifdef GGML_USE_MPI - ggml_mpi_context * ctx_mpi = NULL; -#endif - - int buf_last = 0; - size_t buf_max_size[LLAMA_MAX_SCRATCH_BUFFERS] = { 0 }; - - void use_buf(struct ggml_context * ctx, int i) { -#if defined(LLAMA_USE_SCRATCH) - size_t last_size = 0; - - if (i == -1) { - last_size = ggml_set_scratch(ctx, { 0, 0, nullptr, }); - } else { - auto & buf = buf_scratch[i]; - last_size = ggml_set_scratch(ctx, { 0, buf.size, buf.addr, }); - } - - if (buf_last >= 0) { - buf_max_size[buf_last] = std::max(buf_max_size[buf_last], last_size); - } - - buf_last = i; -#else - (void) i; - (void) ctx; -#endif - } - - size_t get_buf_max_mem(int i) const { -#if defined(LLAMA_USE_SCRATCH) - return buf_max_size[i]; -#else - (void) i; - return 0; -#endif - } -}; - -template -static T checked_mul(T a, T b) { - T ret = a * b; - if (a != 0 && ret / a != b) { - throw std::runtime_error(format("overflow multiplying %llu * %llu", - (unsigned long long) a, (unsigned long long) b)); - } - return ret; -} - -static size_t checked_div(size_t a, size_t b) { - if (b == 0 || a % b != 0) { - throw std::runtime_error(format("error dividing %zu / %zu", a, b)); - } - return a / b; -} - -static std::string llama_format_tensor_shape(const std::vector & ne) { - char buf[256]; - snprintf(buf, sizeof(buf), "%5u", ne.at(0)); - for (size_t i = 1; i < ne.size(); i++) { - snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), " x %5u", ne.at(i)); - } - return buf; -} - -static size_t llama_calc_tensor_size(const std::vector & ne, enum ggml_type type) { - size_t size = ggml_type_size(type); - for (uint32_t dim : ne) { - size = checked_mul(size, dim); - } - return size / ggml_blck_size(type); -} - -struct gguf_load_tensor { - std::string name; - enum ggml_type type = GGML_TYPE_F32; - std::vector ne; - size_t file_off; - size_t size; - struct ggml_tensor * ggml_tensor = NULL; - uint8_t * data; -}; - -struct gguf_load_tensors_map { - // tensors is kept in a separate vector to preserve file order - std::vector tensors; - std::unordered_map name_to_idx; -}; - -enum gguf_file_version { - GGUF_FILE_VERSION_V1 = 1, - -}; - - -struct gguf_file_loader { - gguf_file file; - gguf_context * gguf_ctx; - gguf_file_version file_version; - llama_hparams hparams; - llama_vocab vocab; -struct ggml_context * ctx_data = NULL; - - gguf_file_loader(const char * fname, gguf_load_tensors_map & tensors_map) - : file(fname, "rb") { - fprintf(stderr, "llama.cpp: loading model from %s\n", fname); - - struct gguf_init_params params = { - /*.no_alloc = */ true, - /*.ctx = */ &ctx_data, - }; - - gguf_ctx = gguf_init_from_file(fname, params); - file_version = (enum gguf_file_version) gguf_get_version(gguf_ctx); - - read_hparams(); - read_vocab(); - read_tensor_metadata(tensors_map); - } - - uint32_t read_u32(const char * key) { - int i = gguf_find_key(gguf_ctx, key); - if (i == -1) { - throw std::runtime_error(format("cannot find param with key %s\n", key)); - } - - return gguf_get_val_u32(gguf_ctx, i); - } - - float read_f32(const char * key) { - int i = gguf_find_key(gguf_ctx, key); - if (i == -1) { - throw std::runtime_error(format("cannot find param with key %s\n", key)); - } - - return gguf_get_val_f32(gguf_ctx, i); - } - - int read_n_vocab() { - int i = gguf_find_key(gguf_ctx, "tokenizer.ggml.tokens"); - if (i == -1) { - throw std::runtime_error("cannot find token list in GGUF file\n"); - } - - return gguf_get_arr_n(gguf_ctx, i); - } - - void read_hparams() { - - // TODO define keys as constants in header - // TODO: read all hparams from file - - hparams.n_vocab = read_n_vocab(); - hparams.n_ctx = read_u32("llama.context_length"); - hparams.n_embd = read_u32("llama.embedding_length"); - hparams.n_ff = read_u32("llama.feed_forward_length"); - hparams.n_head = read_u32("llama.attention.head_count"); - hparams.n_layer = read_u32("llama.layer_count"); - hparams.n_rot = read_u32("llama.rope.dimension_count"); - hparams.f_rms_norm_eps = read_f32("llama.attention.layer_norm_rms_epsilon"); - - // LLaMAv2 - // hparams.n_head_kv = read_u32("llama.attention.head_count_kv"); - } - - void read_vocab() { - vocab.id_to_token.resize(hparams.n_vocab); - int token_idx = gguf_find_key(gguf_ctx, "tokenizer.ggml.tokens"); - if (token_idx == -1) { - throw std::runtime_error("cannot find token list in GGUF file\n"); - } - - int score_idx = gguf_find_key(gguf_ctx, "tokenizer.ggml.scores"); - if (score_idx == -1) { - throw std::runtime_error("cannot find token scores list in GGUF file\n"); - } - - for (uint32_t i = 0; i < hparams.n_vocab; i++) { - - std::string word = gguf_get_arr_str(gguf_ctx, token_idx, i); - - vocab.token_to_id[word] = i; - - auto & tok_score = vocab.id_to_token[i]; - tok_score.tok = std::move(word); - tok_score.score = gguf_get_arr_f32(gguf_ctx, score_idx, i); - } - } - - void read_tensor_metadata(gguf_load_tensors_map & tensors_map) { - const int n_tensors = gguf_get_n_tensors(gguf_ctx); - - for (int i = 0; i < n_tensors; ++i) { - gguf_load_tensor tensor; - const char * name = gguf_get_tensor_name(gguf_ctx, i); - - struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); - uint32_t n_dims = cur->n_dims; - tensor.type = cur->type; - tensor.ne.resize(n_dims); - for (uint32_t j = 0; j < n_dims; ++j) { - tensor.ne[j] = cur->ne[j]; - } - - if (n_dims < 1 || n_dims > 2) { - throw std::runtime_error(format("llama.cpp: tensor '%s' should not be %u-dimensional", name, n_dims)); - } - switch (tensor.type) { - case GGML_TYPE_F32: - case GGML_TYPE_F16: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - break; - default: { - throw std::runtime_error(format("unrecognized tensor type %u\n", tensor.type)); - } - } - - - tensor.file_off = gguf_get_data_offset(gguf_ctx) + gguf_get_tensor_offset(gguf_ctx, i); - - tensor.name = name; - tensor.size = llama_calc_tensor_size(tensor.ne, tensor.type); - - tensors_map.tensors.push_back(tensor); - tensors_map.name_to_idx[name] = tensors_map.tensors.size() - 1; - } - } -}; - -struct gguf_file_saver { - // TODO - // this implementation now assumes that the data section is of the same length as the unquantized model. - // this is needed to write tensor metadata and weights in a single pass by seeking to appropriate positions in the file. - // this may not be true when we add quantization version and change ftype description (currently it's string according to the specs, - // but better to have it as uint32). - // we need to calculate the delta in number of bytes written with a counter as a struct member. - - gguf_file file; - gguf_file_loader * fl; - size_t info_offset; - size_t tensor_offset = 0; - - gguf_file_saver(const char * fname, gguf_file_loader * fl, enum llama_ftype new_ftype) - : file(fname, "wb"), fl(fl) { - fprintf(stderr, "llama.cpp: saving model to %s\n", fname); - write_header(); - write_hparams(new_ftype); - } - - void write_header() { - const int32_t magic = GGUF_MAGIC; - file.write_i32(magic); - - const int32_t version = GGUF_VERSION; - file.write_i32(version); - - const int32_t n_tensors = gguf_get_n_tensors(fl->gguf_ctx); - file.write_i32(n_tensors); - - const int32_t n_kv = gguf_get_n_kv(fl->gguf_ctx); - file.write_i32(n_kv); - } - - void write_hparam_arr_str(const std::string & key, enum gguf_type type, int i, int n_arr) { - std::vector data(n_arr); - - for (int j = 0; j < n_arr; ++j) { - std::string val = gguf_get_arr_str(fl->gguf_ctx, i, j); - data[j] = val; - } - - file.write_arr(key, type, data); - } - - void write_hparam_arr_f32(const std::string & key, enum gguf_type type, int i, int n_arr) { - std::vector data(n_arr); - - for (int j = 0; j < n_arr; ++j) { - float val = gguf_get_arr_f32(fl->gguf_ctx, i, j); - data[j] = val; - } - - file.write_arr(key, type, data); - } - - void write_hparams(enum llama_ftype new_ftype) { - const int32_t n_kv = gguf_get_n_kv(fl->gguf_ctx); - for (int i = 0; i < n_kv; ++i) { - const char * key = gguf_get_key(fl->gguf_ctx, i); - if (strcmp(key, "general.quantization_version") == 0) { - file.write_val("general.quantization_version", GGUF_TYPE_UINT32, new_ftype); - } else { - const gguf_type vtype = gguf_get_kv_type(fl->gguf_ctx, i); - - bool bool_val; - float f32_val; - int16_t i16_val; - int32_t i32_val; - int8_t i8_val; - std::string str_val; - uint16_t u16_val; - uint32_t u32_val; - uint8_t u8_val; - gguf_type arr_type; - int n_arr; - - switch(vtype) { - case GGUF_TYPE_BOOL: - bool_val = gguf_get_val_bool(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_BOOL, bool_val); - break; - case GGUF_TYPE_FLOAT32: - f32_val = gguf_get_val_f32(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_FLOAT32, f32_val); - break; - case GGUF_TYPE_INT16: - i16_val = gguf_get_val_i16(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_INT16, i16_val); - break; - case GGUF_TYPE_INT32: - i32_val = gguf_get_val_i32(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_INT32, i32_val); - break; - case GGUF_TYPE_INT8: - i8_val = gguf_get_val_i8(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_INT8, i8_val); - break; - case GGUF_TYPE_STRING: - str_val = gguf_get_val_str(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_STRING, str_val); - break; - case GGUF_TYPE_UINT16: - u16_val = gguf_get_val_u16(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_UINT16, u16_val); - break; - case GGUF_TYPE_UINT32: - u32_val = gguf_get_val_u32(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_UINT32, u32_val); - break; - case GGUF_TYPE_UINT8: - u8_val = gguf_get_val_u8(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_UINT8, u8_val); - break; - case GGUF_TYPE_ARRAY: - arr_type = gguf_get_arr_type(fl->gguf_ctx, i); - n_arr = gguf_get_arr_n(fl->gguf_ctx, i); - if (arr_type == GGUF_TYPE_FLOAT32) { - write_hparam_arr_f32(key, arr_type, i, n_arr); - } else if (arr_type == GGUF_TYPE_STRING) { - write_hparam_arr_str(key, GGUF_TYPE_STRING, i, n_arr); - } else { - throw std::runtime_error("not implemented"); - } - break; - default: - throw std::runtime_error(format("cannot recognize value type for key %s\n", key)); - } - } - } - - info_offset = file.tell(); - size_t count = gguf_get_data_offset(fl->gguf_ctx) - info_offset; - file.write_zeros(count); - file.seek(info_offset, SEEK_SET); - GGML_ASSERT(info_offset == file.tell()); - } - - size_t write_tensor_info(gguf_load_tensor & tensor, enum ggml_type type) { - size_t total_written = 0; - file.seek(info_offset, SEEK_SET); - GGML_ASSERT(info_offset == file.tell()); - total_written += file.write_str(tensor.name); - - int32_t n_dims = tensor.ne.size(); - total_written += file.write_i32(n_dims); - for (int32_t i = 0; i < n_dims; ++i) { - total_written += file.write_i32(tensor.ne[i]); - } - - total_written += file.write_i32(type); - total_written += file.write_u64(tensor_offset); - info_offset += total_written; // position to write info of the next tensor - - file.seek(0, SEEK_END); - - return total_written; - } - - void write_tensor(gguf_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { - switch (new_type) { - case GGML_TYPE_F32: - case GGML_TYPE_F16: - case GGML_TYPE_Q4_0: - case GGML_TYPE_Q4_1: - case GGML_TYPE_Q5_0: - case GGML_TYPE_Q5_1: - case GGML_TYPE_Q8_0: - case GGML_TYPE_Q2_K: - case GGML_TYPE_Q3_K: - case GGML_TYPE_Q4_K: - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - break; - default: GGML_ASSERT(false); - } - - write_tensor_info(tensor, new_type); - file.write_raw(new_data, new_size); - size_t padded_size = GGML_PAD(new_size, GGUF_DEFAULT_ALIGNMENT); // TODO: handle custom alignment - size_t pad = padded_size - new_size; - file.write_zeros(pad); - tensor_offset += padded_size; // offset of the next tensor - } -}; - -struct llama_model_loader { - std::unique_ptr file_loader; - gguf_load_tensors_map tensors_map; - bool use_mmap; - size_t num_ggml_tensors_created = 0; - struct ggml_context * ggml_ctx = NULL; - std::unique_ptr mapping; - - llama_model_loader(const std::string & fname_base, bool use_mmap) { - file_loader = std::unique_ptr(new gguf_file_loader(fname_base.c_str(), tensors_map)); - if (!gguf_mmap::SUPPORTED) { - use_mmap = false; - } - this->use_mmap = use_mmap; - } - - void calc_sizes(size_t * ctx_size_p, size_t * mmapped_size_p) const { - *ctx_size_p = *mmapped_size_p = 0; - for (const gguf_load_tensor & lt : tensors_map.tensors) { - *ctx_size_p += sizeof(struct ggml_tensor) + GGML_OBJECT_SIZE; - *(use_mmap ? mmapped_size_p : ctx_size_p) += lt.size + 16; - } - } - - struct ggml_tensor * get_tensor(const std::string & name, const std::vector & ne, ggml_backend backend) { - auto it = tensors_map.name_to_idx.find(name); - if (it == tensors_map.name_to_idx.end()) { - throw std::runtime_error(std::runtime_error(format("llama.cpp: tensor '%s' is missing from model", name.c_str()))); - } - gguf_load_tensor & lt = tensors_map.tensors.at(it->second); - if (lt.ne != ne) { - throw std::runtime_error(format("llama.cpp: tensor '%s' has wrong shape; expected %s, got %s", - name.c_str(), llama_format_tensor_shape(ne).c_str(), llama_format_tensor_shape(lt.ne).c_str())); - } - - return get_tensor_for(lt, backend); - } - - struct ggml_tensor * get_tensor_for(gguf_load_tensor & lt, ggml_backend backend) { - struct ggml_tensor * tensor; - if (backend != GGML_BACKEND_CPU) { - ggml_set_no_alloc(ggml_ctx, true); - } - if (lt.ne.size() == 2) { - tensor = ggml_new_tensor_2d(ggml_ctx, lt.type, lt.ne.at(0), lt.ne.at(1)); - } else { - GGML_ASSERT(lt.ne.size() == 1); - tensor = ggml_new_tensor_1d(ggml_ctx, lt.type, lt.ne.at(0)); - } - ggml_set_name(tensor, lt.name.c_str()); - GGML_ASSERT(lt.ggml_tensor == NULL); // if this fails, we called get_tensor twice on the same tensor - - if (backend != GGML_BACKEND_CPU) { - ggml_set_no_alloc(ggml_ctx, use_mmap); - } - tensor->backend = backend; - lt.ggml_tensor = tensor; - num_ggml_tensors_created++; - return tensor; - } - - void done_getting_tensors() const { - if (num_ggml_tensors_created != tensors_map.tensors.size()) { - throw std::runtime_error(std::string("llama.cpp: file contained more tensors than expected")); - } - } - - void load_all_data(llama_progress_callback progress_callback, void * progress_callback_user_data, gguf_mlock * lmlock) { - size_t data_size = 0; - size_t prefetch_size = 0; - size_t lock_size = 0; - for (const gguf_load_tensor & lt : tensors_map.tensors) { - data_size += lt.size; - if (lt.ggml_tensor->backend == GGML_BACKEND_CPU) { - prefetch_size += lt.size; - } - } - - if (use_mmap) { - mapping.reset(new gguf_mmap(&file_loader->file, prefetch_size, ggml_is_numa())); - if (lmlock) { - lmlock->init(mapping->addr); - } - } - - size_t done_size = 0; - for (gguf_load_tensor & lt : tensors_map.tensors) { - if (progress_callback) { - progress_callback((float) done_size / data_size, progress_callback_user_data); - } - GGML_ASSERT(lt.ggml_tensor); // unused tensors should have been caught by load_data already - lt.data = (uint8_t *) lt.ggml_tensor->data; - - // allocate temp buffer if not using mmap - if (!use_mmap && lt.data == NULL) { - GGML_ASSERT(lt.ggml_tensor->backend != GGML_BACKEND_CPU); - lt.data = (uint8_t*)malloc(ggml_nbytes(lt.ggml_tensor)); - } - - load_data_for(lt); - - switch(lt.ggml_tensor->backend) { - case GGML_BACKEND_CPU: - lt.ggml_tensor->data = lt.data; - if (use_mmap && lmlock) { - lock_size += lt.size; - lmlock->grow_to(lock_size); - } - break; -#if defined(GGML_USE_CUBLAS) - case GGML_BACKEND_GPU: - case GGML_BACKEND_GPU_SPLIT: - ggml_cuda_transform_tensor(lt.data, lt.ggml_tensor); - if (!use_mmap) { - free(lt.data); - } - break; -#elif defined(GGML_USE_CLBLAST) - case GGML_BACKEND_GPU: - ggml_cl_transform_tensor(lt.data, lt.ggml_tensor); - if (!use_mmap) { - free(lt.data); - } - break; -#endif - default: - continue; - } - - done_size += lt.size; - } - } - - void load_data_for(gguf_load_tensor & lt) { - if (use_mmap) { - lt.data = (uint8_t *) mapping->addr + lt.file_off; - } else { - gguf_file & file = file_loader->file; - file.seek(lt.file_off, SEEK_SET); - file.read_raw(lt.data, lt.size); - } - - if (0) { - print_checksum(lt); - } - } - - static void print_checksum(gguf_load_tensor & lt) { - uint32_t sum = 0; - for (size_t i = 0; i < lt.size; i++) { - uint8_t byte = lt.data[i]; - sum = byte + (sum << 6) + (sum << 16) - sum; // sdbm hash - } - fprintf(stderr, "%s checksum: %#08x (%s, size %zu)\n", lt.name.c_str(), sum, - llama_format_tensor_shape(lt.ne).c_str(), lt.size); - } - -}; - -// -// kv cache -// - -static bool kv_cache_init( - const struct llama_hparams & hparams, - struct llama_kv_cache & cache, - ggml_type wtype, - int n_ctx, - int n_gpu_layers) { - const int n_embd = hparams.n_embd_gqa(); - const int n_layer = hparams.n_layer; - - const int64_t n_mem = n_layer*n_ctx; - const int64_t n_elements = n_embd*n_mem; - - cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB); - cache.n = 0; - - struct ggml_init_params params; - params.mem_size = cache.buf.size; - params.mem_buffer = cache.buf.addr; - params.no_alloc = false; - - cache.ctx = ggml_init(params); - - if (!cache.ctx) { - fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__); - return false; - } - - cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements); - cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements); - ggml_set_name(cache.k, "cache_k"); - ggml_set_name(cache.v, "cache_v"); - - (void) n_gpu_layers; -#ifdef GGML_USE_CUBLAS - if (n_gpu_layers > n_layer + 1) { - ggml_cuda_assign_buffers_no_scratch(cache.v); - } - if (n_gpu_layers > n_layer + 2) { - ggml_cuda_assign_buffers_no_scratch(cache.k); - } -#endif // GGML_USE_CUBLAS - - return true; -} - -struct llama_context_params llama_context_default_params() { - struct llama_context_params result = { - /*.seed =*/ LLAMA_DEFAULT_SEED, - /*.n_ctx =*/ 512, - /*.n_batch =*/ 512, - /*.n_gqa =*/ 1, - /*.rms_norm_eps =*/ LLAMA_DEFAULT_RMS_EPS, - /*.gpu_layers =*/ 0, - /*.main_gpu =*/ 0, - /*.tensor_split =*/ nullptr, - /*.rope_freq_base =*/ 10000.0f, - /*.rope_freq_scale =*/ 1.0f, - /*.progress_callback =*/ nullptr, - /*.progress_callback_user_data =*/ nullptr, - /*.low_vram =*/ false, - /*.f16_kv =*/ true, - /*.logits_all =*/ false, - /*.vocab_only =*/ false, - /*.use_mmap =*/ true, - /*.use_mlock =*/ false, - /*.embedding =*/ false, - }; - - return result; -} - -struct llama_model_quantize_params llama_model_quantize_default_params() { - struct llama_model_quantize_params result = { - /*.nthread =*/ 0, - /*.ftype =*/ LLAMA_FTYPE_MOSTLY_Q5_1, - /*.allow_requantize =*/ false, - /*.quantize_output_tensor =*/ true, - }; - - return result; -} - -int llama_max_devices() { - return LLAMA_MAX_DEVICES; -} - -bool llama_mmap_supported() { - return gguf_mmap::SUPPORTED; -} - -bool llama_mlock_supported() { - return gguf_mlock::SUPPORTED; -} - -void llama_backend_init(bool numa) { - ggml_time_init(); - - // needed to initialize f16 tables - { - struct ggml_init_params params = { 0, NULL, false }; - struct ggml_context * ctx = ggml_init(params); - ggml_free(ctx); - } - - if (numa) { - ggml_numa_init(); - } - -#ifdef GGML_USE_MPI - ggml_mpi_backend_init(); -#endif -} - -void llama_backend_free() { -#ifdef GGML_USE_MPI - ggml_mpi_backend_free(); -#endif -} - -int64_t llama_time_us() { - return ggml_time_us(); -} - -// -// model loading -// - -static const char *gguf_file_version_name(gguf_file_version version) { - switch (version) { - case GGUF_FILE_VERSION_V1: return "GGUF V1 (latest)"; - } - - return "unknown"; -} - -static const char *llama_ftype_name(enum llama_ftype ftype) { - switch (ftype) { - case LLAMA_FTYPE_ALL_F32: return "all F32"; - case LLAMA_FTYPE_MOSTLY_F16: return "mostly F16"; - case LLAMA_FTYPE_MOSTLY_Q4_0: return "mostly Q4_0"; - case LLAMA_FTYPE_MOSTLY_Q4_1: return "mostly Q4_1"; - case LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16: - return "mostly Q4_1, some F16"; - case LLAMA_FTYPE_MOSTLY_Q5_0: return "mostly Q5_0"; - case LLAMA_FTYPE_MOSTLY_Q5_1: return "mostly Q5_1"; - case LLAMA_FTYPE_MOSTLY_Q8_0: return "mostly Q8_0"; - // K-quants - case LLAMA_FTYPE_MOSTLY_Q2_K: return "mostly Q2_K"; - case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "mostly Q3_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "mostly Q3_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "mostly Q3_K - Large"; - case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "mostly Q4_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "mostly Q4_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "mostly Q5_K - Small"; - case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "mostly Q5_K - Medium"; - case LLAMA_FTYPE_MOSTLY_Q6_K: return "mostly Q6_K"; - default: return "unknown, may not work"; - } -} - -static const char *llama_model_type_name(e_model type) { - switch (type) { - case MODEL_3B: return "3B"; - case MODEL_7B: return "7B"; - case MODEL_13B: return "13B"; - case MODEL_30B: return "30B"; - case MODEL_65B: return "65B"; - case MODEL_70B: return "70B"; - default: GGML_ASSERT(false); - } -} - -static void llama_model_load_internal( - const std::string & fname, - llama_model & model, - llama_vocab & vocab, - int n_ctx, - int n_batch, - int n_gqa, - float rms_norm_eps, - int n_gpu_layers, - int main_gpu, - const float * tensor_split, - float rope_freq_base, - float rope_freq_scale, - bool low_vram, - ggml_type memory_type, - bool use_mmap, - bool use_mlock, - bool vocab_only, - llama_progress_callback progress_callback, - void * progress_callback_user_data) { - GGML_UNUSED(rms_norm_eps); // TODO: update function signature to remove this - - model.t_start_us = ggml_time_us(); - - std::unique_ptr ml(new llama_model_loader(fname, use_mmap)); - - vocab = std::move(ml->file_loader->vocab); - model.hparams = ml->file_loader->hparams; - model.n_gpu_layers = n_gpu_layers; - gguf_file_version file_version = ml->file_loader->file_version; - - auto & hparams = model.hparams; - - { - switch (hparams.n_layer) { - case 26: model.type = e_model::MODEL_3B; break; - case 32: model.type = e_model::MODEL_7B; break; - case 40: model.type = e_model::MODEL_13B; break; - case 60: model.type = e_model::MODEL_30B; break; - case 80: model.type = e_model::MODEL_65B; break; - default: - { - if (hparams.n_layer < 32) { - model.type = e_model::MODEL_7B; - } - } break; - } - - hparams.n_ctx = n_ctx; - - // LLaMAv2 - hparams.n_head_kv = hparams.n_head / n_gqa; - if (model.type == e_model::MODEL_65B && n_gqa == 8) { - fprintf(stderr, "%s: warning: assuming 70B model based on GQA == %d\n", __func__, n_gqa); - model.type = e_model::MODEL_70B; - } - - hparams.rope_freq_base = rope_freq_base; - hparams.rope_freq_scale = rope_freq_scale; - } - - const uint32_t n_ff = hparams.n_ff; - - { - fprintf(stderr, "%s: format = %s\n", __func__, gguf_file_version_name(file_version)); - fprintf(stderr, "%s: n_vocab = %u\n", __func__, hparams.n_vocab); - fprintf(stderr, "%s: n_ctx = %u\n", __func__, hparams.n_ctx); - fprintf(stderr, "%s: n_embd = %u\n", __func__, hparams.n_embd); - fprintf(stderr, "%s: n_head = %u\n", __func__, hparams.n_head); - fprintf(stderr, "%s: n_head_kv = %u\n", __func__, hparams.n_head_kv); - fprintf(stderr, "%s: n_layer = %u\n", __func__, hparams.n_layer); - fprintf(stderr, "%s: n_rot = %u\n", __func__, hparams.n_rot); // a.k.a. n_embd_head, n_head_dim - fprintf(stderr, "%s: n_gqa = %u\n", __func__, hparams.n_gqa()); - fprintf(stderr, "%s: rnorm_eps = %.1e\n", __func__, hparams.f_rms_norm_eps); - fprintf(stderr, "%s: n_ff = %u\n", __func__, n_ff); - fprintf(stderr, "%s: freq_base = %.1f\n", __func__, hparams.rope_freq_base); - fprintf(stderr, "%s: freq_scale = %g\n", __func__, hparams.rope_freq_scale); - fprintf(stderr, "%s: ftype = %u (%s)\n", __func__, hparams.ftype, llama_ftype_name(hparams.ftype)); - fprintf(stderr, "%s: model size = %s\n", __func__, llama_model_type_name(model.type)); - } - - if (hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || - hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_1 || - hparams.ftype == LLAMA_FTYPE_MOSTLY_Q8_0) { - throw std::runtime_error(format("this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1508)")); - } - - if (vocab_only) { - return; - } - - auto & ctx = model.ctx; - - size_t ctx_size; - size_t mmapped_size; - ml->calc_sizes(&ctx_size, &mmapped_size); - fprintf(stderr, "%s: ggml ctx size = %7.2f MB\n", __func__, ctx_size/1024.0/1024.0); - - // create the ggml context - { - model.buf.resize(ctx_size); - if (use_mlock) { - model.mlock_buf.init (model.buf.addr); - model.mlock_buf.grow_to(model.buf.size); - } - - struct ggml_init_params params = { - /*.mem_size =*/ model.buf.size, - /*.mem_buffer =*/ model.buf.addr, - /*.no_alloc =*/ ml->use_mmap, - }; - - model.ctx = ggml_init(params); - if (!model.ctx) { - throw std::runtime_error(format("ggml_init() failed")); - } - } - - (void) main_gpu; -#if defined(GGML_USE_CUBLAS) - fprintf(stderr, "%s: using CUDA for GPU acceleration\n", __func__); - ggml_cuda_set_main_device(main_gpu); -#define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU -#define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_GPU_SPLIT -#elif defined(GGML_USE_CLBLAST) - fprintf(stderr, "%s: using OpenCL for GPU acceleration\n", __func__); -#define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU -#define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_GPU -#else -#define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_CPU -#define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_CPU -#endif - - // prepare memory for the weights - size_t vram_weights = 0; - size_t vram_scratch = 0; - { - const uint32_t n_embd = hparams.n_embd; - const uint32_t n_embd_gqa = hparams.n_embd_gqa(); - const uint32_t n_layer = hparams.n_layer; - const uint32_t n_vocab = hparams.n_vocab; - - ml->ggml_ctx = ctx; - - model.tok_embeddings = ml->get_tensor("tok_embeddings.weight", {n_embd, n_vocab}, GGML_BACKEND_CPU); - - // "output" tensor - { - ggml_backend backend_norm; - ggml_backend backend_output; - if (n_gpu_layers > int(n_layer)) { // NOLINT - // norm is not performance relevant on its own but keeping it in VRAM reduces data copying - // on Windows however this is detrimental unless everything is on the GPU -#ifndef _WIN32 - backend_norm = low_vram ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD; -#else - backend_norm = low_vram || n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD; -#endif // _WIN32 - - backend_output = LLAMA_BACKEND_OFFLOAD_SPLIT; - } else { - backend_norm = GGML_BACKEND_CPU; - backend_output = GGML_BACKEND_CPU; - } - - model.norm = ml->get_tensor("norm.weight", {n_embd}, backend_norm); - model.output = ml->get_tensor("output.weight", {n_embd, n_vocab}, backend_output); - if (backend_norm == GGML_BACKEND_GPU) { - vram_weights += ggml_nbytes(model.norm); - } - if (backend_output == GGML_BACKEND_GPU_SPLIT) { - vram_weights += ggml_nbytes(model.output); - } - } - - const int i_gpu_start = n_layer - n_gpu_layers; - - model.layers.resize(n_layer); - for (uint32_t i = 0; i < n_layer; ++i) { - const ggml_backend backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD; // NOLINT - const ggml_backend backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD_SPLIT; // NOLINT - - auto & layer = model.layers[i]; - - std::string layers_i = "layers." + std::to_string(i); - - layer.attention_norm = ml->get_tensor(layers_i + ".attention_norm.weight", {n_embd}, backend); - - layer.wq = ml->get_tensor(layers_i + ".attention.wq.weight", {n_embd, n_embd}, backend_split); - layer.wk = ml->get_tensor(layers_i + ".attention.wk.weight", {n_embd, n_embd_gqa}, backend_split); - layer.wv = ml->get_tensor(layers_i + ".attention.wv.weight", {n_embd, n_embd_gqa}, backend_split); - layer.wo = ml->get_tensor(layers_i + ".attention.wo.weight", {n_embd, n_embd}, backend_split); - - layer.ffn_norm = ml->get_tensor(layers_i + ".ffn_norm.weight", {n_embd}, backend); - - layer.w1 = ml->get_tensor(layers_i + ".feed_forward.w1.weight", {n_embd, n_ff}, backend_split); - layer.w2 = ml->get_tensor(layers_i + ".feed_forward.w2.weight", { n_ff, n_embd}, backend_split); - layer.w3 = ml->get_tensor(layers_i + ".feed_forward.w3.weight", {n_embd, n_ff}, backend_split); - - if (backend == GGML_BACKEND_GPU) { - vram_weights += - ggml_nbytes(layer.attention_norm) + ggml_nbytes(layer.wq) + ggml_nbytes(layer.wk) + - ggml_nbytes(layer.wv) + ggml_nbytes(layer.wo) + ggml_nbytes(layer.ffn_norm) + - ggml_nbytes(layer.w1) + ggml_nbytes(layer.w2) + ggml_nbytes(layer.w3); - } - } - } - - ml->done_getting_tensors(); - - // print memory requirements - { - const size_t scale = memory_type == GGML_TYPE_F32 ? 2 : 1; - - // this is the total memory required to run the inference - const size_t mem_required = - ctx_size + - mmapped_size - vram_weights + // weights in VRAM not in memory - MEM_REQ_SCRATCH0(hparams.n_ctx).at(model.type) + - MEM_REQ_SCRATCH1().at(model.type) + - MEM_REQ_EVAL().at(model.type); - - // this is the memory required by one llama_state - const size_t mem_required_state = - scale*hparams.kv_size(); - - fprintf(stderr, "%s: mem required = %7.2f MB (+ %7.2f MB per state)\n", __func__, - mem_required / 1024.0 / 1024.0, mem_required_state / 1024.0 / 1024.0); - - (void) vram_scratch; - (void) n_batch; -#ifdef GGML_USE_CUBLAS - if (low_vram) { - fprintf(stderr, "%s: not allocating a VRAM scratch buffer due to low VRAM option\n", __func__); - ggml_cuda_set_scratch_size(0); // disable scratch - } else { - const size_t vram_scratch_base = VRAM_REQ_SCRATCH_BASE().at(model.type); - const size_t vram_scratch_per_context = VRAM_REQ_SCRATCH_PER_CONTEXT().at(model.type); - vram_scratch = n_batch * (vram_scratch_base + n_ctx * vram_scratch_per_context); - ggml_cuda_set_scratch_size(vram_scratch); - if (n_gpu_layers > 0) { - fprintf(stderr, "%s: allocating batch_size x (%zd kB + n_ctx x %zd B) = %zd MB VRAM for the scratch buffer\n", - __func__, vram_scratch_base / kB, vram_scratch_per_context, - (vram_scratch + MB - 1) / MB); // round up - } - } -#endif // GGML_USE_CUBLAS - -#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) - const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer)); - - fprintf(stderr, "%s: offloading %d repeating layers to GPU\n", __func__, n_gpu); - if (n_gpu_layers > (int) hparams.n_layer) { - fprintf(stderr, "%s: offloading non-repeating layers to GPU\n", __func__); - } - size_t vram_kv_cache = 0; - -#ifdef GGML_USE_CUBLAS - const int max_backend_supported_layers = hparams.n_layer + 3; - const int max_offloadable_layers = low_vram ? hparams.n_layer + 1 : hparams.n_layer + 3; - if (n_gpu_layers > (int) hparams.n_layer + 1) { - if (low_vram) { - fprintf(stderr, "%s: cannot offload v cache to GPU due to low VRAM option\n", __func__); - } else { - fprintf(stderr, "%s: offloading v cache to GPU\n", __func__); - vram_kv_cache += hparams.kv_size() / 2; - } - } - if (n_gpu_layers > (int) hparams.n_layer + 2) { - if (low_vram) { - fprintf(stderr, "%s: cannot offload k cache to GPU due to low VRAM option\n", __func__); - } else { - fprintf(stderr, "%s: offloading k cache to GPU\n", __func__); - vram_kv_cache += hparams.kv_size() / 2; - } - } -#elif defined(GGML_USE_CLBLAST) - const int max_backend_supported_layers = hparams.n_layer + 1; - const int max_offloadable_layers = hparams.n_layer + 1; -#endif // GGML_USE_CUBLAS - - fprintf(stderr, "%s: offloaded %d/%d layers to GPU\n", - __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers); - fprintf(stderr, "%s: total VRAM used: %zu MB\n", - __func__, (vram_weights + vram_scratch + vram_kv_cache + MB - 1) / MB); // round up -#else - (void) n_gpu_layers; -#endif // defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) - } - - // populate `tensors_by_name` - for (gguf_load_tensor & lt : ml->tensors_map.tensors) { - model.tensors_by_name.emplace_back(lt.name, lt.ggml_tensor); - } - - (void) tensor_split; -#if defined(GGML_USE_CUBLAS) - { - ggml_cuda_set_tensor_split(tensor_split); - } -#endif - - ml->load_all_data(progress_callback, progress_callback_user_data, use_mlock ? &model.mlock_mmap : NULL); - - if (progress_callback) { - progress_callback(1.0f, progress_callback_user_data); - } - - model.mapping = std::move(ml->mapping); - - // loading time will be recalculate after the first eval, so - // we take page faults deferred by mmap() into consideration - model.t_load_us = ggml_time_us() - model.t_start_us; -} - -static bool llama_model_load( - const std::string & fname, - llama_model & model, - llama_vocab & vocab, - int n_ctx, - int n_batch, - int n_gqa, - float rms_norm_eps, - int n_gpu_layers, - int main_gpu, - const float * tensor_split, - float rope_freq_base, - float rope_freq_scale, - bool low_vram, - ggml_type memory_type, - bool use_mmap, - bool use_mlock, - bool vocab_only, - llama_progress_callback progress_callback, - void *progress_callback_user_data) { - try { - llama_model_load_internal(fname, model, vocab, n_ctx, n_batch, n_gqa, rms_norm_eps, n_gpu_layers, main_gpu, tensor_split, rope_freq_base, rope_freq_scale, low_vram, memory_type, - use_mmap, use_mlock, vocab_only, progress_callback, progress_callback_user_data); - return true; - } catch (const std::exception & err) { - fprintf(stderr, "error loading model: %s\n", err.what()); - return false; - } -} - -// evaluate the transformer -// -// - lctx: llama context -// - tokens: new batch of tokens to process -// - embd embeddings input -// - n_tokens number of tokens -// - n_past: the context size so far -// - n_threads: number of threads to use -// -static bool llama_eval_internal( - llama_context & lctx, - const llama_token * tokens, - const float * embd, - int n_tokens, - int n_past, - int n_threads, - const char * cgraph_fname) { - - GGML_ASSERT((!tokens && embd) || (tokens && !embd)); - -#ifdef GGML_USE_MPI - ggml_mpi_eval_init(lctx.ctx_mpi, &n_tokens, &n_past, &n_threads); -#endif - - const int64_t t_start_us = ggml_time_us(); - - const int N = n_tokens; - - const auto & model = lctx.model; - const auto & hparams = model.hparams; - - const auto & kv_self = lctx.kv_self; - - GGML_ASSERT(!!kv_self.ctx); - - const int64_t n_embd = hparams.n_embd; - const int64_t n_layer = hparams.n_layer; - const int64_t n_ctx = hparams.n_ctx; - const int64_t n_head = hparams.n_head; - const int64_t n_head_kv = hparams.n_head_kv; - const int64_t n_embd_head = hparams.n_embd_head(); - const int64_t n_vocab = hparams.n_vocab; - const int64_t n_embd_gqa = hparams.n_embd_gqa(); - - - GGML_ASSERT(n_embd_head == hparams.n_rot); - - const float freq_base = hparams.rope_freq_base; - const float freq_scale = hparams.rope_freq_scale; - const float rms_norm_eps = hparams.f_rms_norm_eps; - - const int n_gpu_layers = model.n_gpu_layers; - - auto & mem_per_token = lctx.mem_per_token; - auto & buf_compute = lctx.buf_compute; - - struct ggml_init_params params = { - /*.mem_size =*/ buf_compute.size, - /*.mem_buffer =*/ buf_compute.addr, - /*.no_alloc =*/ false, - }; - - struct ggml_context * ctx0 = ggml_init(params); - - ggml_cgraph * gf = ggml_new_graph(ctx0); - - // for big prompts, if BLAS is enabled, it is better to use only one thread - // otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance - n_threads = N >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas() ? 1 : n_threads; - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - if (tokens) { - struct ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); - memcpy(inp_tokens->data, tokens, N*ggml_element_size(inp_tokens)); - ggml_set_name(inp_tokens, "inp_tokens"); - - inpL = ggml_get_rows(ctx0, model.tok_embeddings, inp_tokens); - } else { -#ifdef GGML_USE_MPI - GGML_ASSERT(false && "not implemented"); -#endif - - inpL = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N); - memcpy(inpL->data, embd, N * n_embd * ggml_element_size(inpL)); - } - - const int i_gpu_start = n_layer - n_gpu_layers; - (void) i_gpu_start; - - // offload functions set the tensor output backend to GPU - // tensors are GPU-accelerated if any input or the output has been offloaded - // - // with the low VRAM option VRAM scratch is disabled in llama_load_model_internal - // in that case ggml_cuda_assign_buffers has no effect - offload_func_t offload_func_nr = llama_nop; // nr = non-repeating - offload_func_t offload_func_kq = llama_nop; - offload_func_t offload_func_v = llama_nop; - -#ifdef GGML_USE_CUBLAS - if (n_gpu_layers > n_layer) { - offload_func_nr = ggml_cuda_assign_buffers; - } - if (n_gpu_layers > n_layer + 1) { - offload_func_v = ggml_cuda_assign_buffers; - } - if (n_gpu_layers > n_layer + 2) { - offload_func_kq = ggml_cuda_assign_buffers; - } -#endif // GGML_USE_CUBLAS - - for (int il = 0; il < n_layer; ++il) { - ggml_format_name(inpL, "layer_inp_%d", il); - - offload_func_t offload_func = llama_nop; - -#ifdef GGML_USE_CUBLAS - if (il >= i_gpu_start) { - offload_func = ggml_cuda_assign_buffers; - } -#endif // GGML_USE_CUBLAS - - struct ggml_tensor * inpSA = inpL; - - lctx.use_buf(ctx0, 0); - - // norm - { - cur = ggml_rms_norm(ctx0, inpL, rms_norm_eps); - offload_func(cur); - ggml_set_name(cur, "rms_norm_0"); - - // cur = cur*attention_norm(broadcasted) - cur = ggml_mul(ctx0, cur, model.layers[il].attention_norm); - offload_func(cur); - ggml_set_name(cur, "attention_norm_0"); - } - - // self-attention - { - // compute Q and K and RoPE them - struct ggml_tensor * tmpk = ggml_mul_mat(ctx0, model.layers[il].wk, cur); - offload_func_kq(tmpk); - ggml_set_name(tmpk, "tmpk"); - - struct ggml_tensor * tmpq = ggml_mul_mat(ctx0, model.layers[il].wq, cur); - offload_func_kq(tmpq); - ggml_set_name(tmpq, "tmpq"); - - struct ggml_tensor * Kcur = ggml_rope_custom_inplace(ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, N), n_past, n_embd_head, 0, 0, freq_base, freq_scale); - offload_func_kq(Kcur); - ggml_set_name(Kcur, "Kcur"); - - struct ggml_tensor * Qcur = ggml_rope_custom_inplace(ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, N), n_past, n_embd_head, 0, 0, freq_base, freq_scale); - offload_func_kq(Qcur); - ggml_set_name(Qcur, "Qcur"); - - // store key and value to memory - { - // compute the transposed [N, n_embd] V matrix - - struct ggml_tensor * tmpv = ggml_mul_mat(ctx0, model.layers[il].wv, cur); - offload_func_v(tmpv); - ggml_set_name(tmpv, "tmpv"); - - struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_reshape_2d(ctx0, tmpv, n_embd_gqa, N)); - offload_func_v(Vcur); - ggml_set_name(Vcur, "Vcur"); - - struct ggml_tensor * k = ggml_view_1d(ctx0, kv_self.k, N*n_embd_gqa, (ggml_element_size(kv_self.k)*n_embd_gqa)*(il*n_ctx + n_past)); - offload_func_kq(k); - ggml_set_name(k, "k"); - - struct ggml_tensor * v = ggml_view_2d(ctx0, kv_self.v, N, n_embd_gqa, - ( n_ctx)*ggml_element_size(kv_self.v), - (il*n_ctx)*ggml_element_size(kv_self.v)*n_embd_gqa + n_past*ggml_element_size(kv_self.v)); - offload_func_v(v); - ggml_set_name(v, "v"); - - // important: storing RoPE-ed version of K in the KV cache! - ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k)); - ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v)); - } - - struct ggml_tensor * Q = - ggml_permute(ctx0, - Qcur, - 0, 2, 1, 3); - offload_func_kq(Q); - ggml_set_name(Q, "Q"); - - struct ggml_tensor * K = - ggml_permute(ctx0, - ggml_reshape_3d(ctx0, - ggml_view_1d(ctx0, kv_self.k, (n_past + N)*n_embd_gqa, il*n_ctx*ggml_element_size(kv_self.k)*n_embd_gqa), - n_embd_head, n_head_kv, n_past + N), - 0, 2, 1, 3); - offload_func_kq(K); - ggml_set_name(K, "K"); - - // K * Q - struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); - offload_func_kq(KQ); - ggml_set_name(KQ, "KQ"); - - // KQ_scaled = KQ / sqrt(n_embd_head) - struct ggml_tensor * KQ_scale = ggml_new_f32(ctx0, 1.0f/sqrtf(float(n_embd)/n_head)); - ggml_set_name(KQ_scale, "1/sqrt(n_embd_head)"); - - // KQ_scaled shape [n_past + N, N, n_head, 1] - struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, KQ_scale); - offload_func_kq(KQ_scaled); - ggml_set_name(KQ_scaled, "KQ_scaled"); - - // KQ_masked = mask_past(KQ_scaled) - struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past); - offload_func_kq(KQ_masked); - ggml_set_name(KQ_masked, "KQ_masked"); - - // KQ = soft_max(KQ_masked) - struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked); - offload_func_v(KQ_soft_max); - ggml_set_name(KQ_soft_max, "KQ_soft_max"); - - // split cached V into n_head heads - struct ggml_tensor * V = - ggml_view_3d(ctx0, kv_self.v, - n_past + N, n_embd_head, n_head_kv, - n_ctx*ggml_element_size(kv_self.v), - n_ctx*ggml_element_size(kv_self.v)*n_embd_head, - n_ctx*ggml_element_size(kv_self.v)*n_embd_gqa*il); - offload_func_v(V); - ggml_set_name(V, "V"); - -#if 1 - struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max); - offload_func_v(KQV); - ggml_set_name(KQV, "KQV"); -#else - // make V contiguous in memory to speed up the matmul, however we waste time on the copy - // on M1 this is faster for the perplexity computation, but ~5% slower for the single-token generation - // is there a better way? - struct ggml_tensor * V_cont = ggml_cpy(ctx0, V, ggml_new_tensor_3d(ctx0, kv_self.v->type, n_past + N, n_embd_head, n_head)); - struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_cont, KQ_soft_max); -#endif - - // KQV_merged = KQV.permute(0, 2, 1, 3) - struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3); - offload_func_v(KQV_merged); - ggml_set_name(KQV_merged, "KQV_merged"); - - // cur = KQV_merged.contiguous().view(n_embd, N) - cur = ggml_cpy(ctx0, - KQV_merged, - ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N)); - offload_func_v(cur); - ggml_set_name(cur, "KQV_merged_contiguous"); - - // projection (no bias) - cur = ggml_mul_mat(ctx0, - model.layers[il].wo, - cur); - offload_func(cur); - ggml_set_name(cur, "result_wo"); - } - - lctx.use_buf(ctx0, 1); - - struct ggml_tensor * inpFF = ggml_add(ctx0, cur, inpSA); - offload_func(inpFF); - ggml_set_name(inpFF, "inpFF"); - - // feed-forward network - { - // norm - { - cur = ggml_rms_norm(ctx0, inpFF, rms_norm_eps); - offload_func(cur); - ggml_set_name(cur, "rms_norm_1"); - - // cur = cur*ffn_norm(broadcasted) - cur = ggml_mul(ctx0, cur, model.layers[il].ffn_norm); - offload_func(cur); - ggml_set_name(cur, "ffn_norm"); - } - - struct ggml_tensor * tmp = ggml_mul_mat(ctx0, - model.layers[il].w3, - cur); - offload_func(tmp); - ggml_set_name(tmp, "result_w3"); - - cur = ggml_mul_mat(ctx0, - model.layers[il].w1, - cur); - offload_func(cur); - ggml_set_name(cur, "result_w1"); - - // SILU activation - cur = ggml_silu(ctx0, cur); - offload_func(cur); - ggml_set_name(cur, "silu"); - - cur = ggml_mul(ctx0, cur, tmp); - offload_func(cur); - ggml_set_name(cur, "silu_x_result_w3"); - - cur = ggml_mul_mat(ctx0, - model.layers[il].w2, - cur); - offload_func(cur); - ggml_set_name(cur, "result_w2"); - } - - cur = ggml_add(ctx0, cur, inpFF); - offload_func(cur); - ggml_set_name(cur, "inpFF_+_result_w2"); - - // input for next layer - inpL = cur; - } - - lctx.use_buf(ctx0, 0); - - // used at the end to optionally extract the embeddings - struct ggml_tensor * embeddings = NULL; - - // norm - { - cur = ggml_rms_norm(ctx0, inpL, rms_norm_eps); - offload_func_nr(cur); - ggml_set_name(cur, "rms_norm_2"); - - // cur = cur*norm(broadcasted) - cur = ggml_mul(ctx0, cur, model.norm); - // offload_func_nr(cur); // TODO CPU + GPU mirrored backend - ggml_set_name(cur, "result_norm"); - - embeddings = cur; - } - - // lm_head - cur = ggml_mul_mat(ctx0, model.output, cur); - ggml_set_name(cur, "result_output"); - - lctx.use_buf(ctx0, -1); - - // logits -> probs - //cur = ggml_soft_max_inplace(ctx0, cur); - - // run the computation - ggml_build_forward_expand(gf, cur); - - // fprintf(stderr, "graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf.n_nodes, gf.n_leafs); - -#if GGML_USE_MPI - ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer); -#endif - -#ifdef GGML_USE_METAL - if (lctx.ctx_metal && N == 1) { - if (!ggml_metal_if_optimized(lctx.ctx_metal)) { - ggml_metal_graph_find_concurrency(lctx.ctx_metal, gf); - } - ggml_metal_set_n_cb (lctx.ctx_metal, n_threads); - ggml_metal_graph_compute(lctx.ctx_metal, gf); - ggml_metal_get_tensor (lctx.ctx_metal, cur); - } else { - // IMPORTANT: - // Since we don't have efficient Matrix x Matrix Metal multiplication yet, we fallback to vanilla - // ggml_graph_compute(). It uses Apple's Accelerate CBLAS API which takes advantage of the ANE or the AMX - // coprocessor. - // - // When we implement Matrix x Matrix Metal multiplication, we can avoid this branch. - // But for now, we have focused only on Matrix x Vector Metal multiplication. - // - // TODO: avoid these syncs via shared memory (ref #1696) - // - if (lctx.ctx_metal) { - // We need to sync the GPU KV cache with the CPU KV cache - ggml_metal_get_tensor(lctx.ctx_metal, kv_self.k); - ggml_metal_get_tensor(lctx.ctx_metal, kv_self.v); - } - - ggml_graph_compute_helper(lctx.work_buffer, gf, n_threads); - } -#else - ggml_graph_compute_helper(lctx.work_buffer, gf, n_threads); -#endif - -#if GGML_USE_MPI - ggml_mpi_graph_compute_post(lctx.ctx_mpi, gf, n_layer); -#endif - - // update kv token count - lctx.kv_self.n = n_past + N; - - struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; - - if (cgraph_fname) { - ggml_graph_export(gf, cgraph_fname); - } - -#ifdef GGML_PERF - // print timing information per ggml operation (for debugging purposes) - // requires GGML_PERF to be defined - ggml_graph_print(gf); -#endif - - // plot the computation graph in dot format (for debugging purposes) - //if (n_past%100 == 0) { - // ggml_graph_dump_dot(gf, NULL, "llama.dot"); - //} - - // extract logits - { - auto & logits_out = lctx.logits; - - if (lctx.logits_all) { - logits_out.resize(n_vocab * N); - memcpy(logits_out.data(), (float *) ggml_get_data(res), sizeof(float)*n_vocab*N); - } else { - // return result for just the last token - logits_out.resize(n_vocab); - memcpy(logits_out.data(), (float *) ggml_get_data(res) + (n_vocab*(N-1)), sizeof(float)*n_vocab); - } - } - - // extract embeddings - if (!lctx.embedding.empty()) { - auto & embedding_out = lctx.embedding; - - embedding_out.resize(n_embd); - memcpy(embedding_out.data(), (float *) ggml_get_data(embeddings) + (n_embd*(N - 1)), sizeof(float)*n_embd); - } - - if (mem_per_token == 0) { - mem_per_token = ggml_used_mem(ctx0)/N; - } - -#if 0 - printf("\n%s: used_mem: eval ctx %.3f MB, scratch %.3f MB %.3f MB, work buf %.3f MB, n_past = %d, N = %d\n", __func__, - ggml_used_mem(ctx0)/1024.0/1024.0, - lctx.get_buf_max_mem(0)/1024.0/1024.0, - lctx.get_buf_max_mem(1)/1024.0/1024.0, - lctx.work_buffer.size()/1024.0/1024.0, - n_past, N); -#endif - - ggml_free(ctx0); - - // measure the performance only for the single-token evals - if (N == 1) { - lctx.t_eval_us += ggml_time_us() - t_start_us; - lctx.n_eval++; - } - else if (N > 1) { - lctx.t_p_eval_us += ggml_time_us() - t_start_us; - lctx.n_p_eval += N; - } - - return true; -} - -// -// tokenizer -// - -static size_t utf8_len(char src) { - const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; - uint8_t highbits = static_cast(src) >> 4; - return lookup[highbits]; -} - -struct llama_sp_symbol { - using index = int; - index prev; - index next; - const char * text; - size_t n; -}; - -static_assert(std::is_trivially_copyable::value, "llama_sp_symbol is not trivially copyable"); - -struct llama_sp_bigram { - struct comparator { - bool operator()(llama_sp_bigram & l, llama_sp_bigram & r) { - return (l.score < r.score) || (l.score == r.score && l.left > r.left); - } - }; - using queue_storage = std::vector; - using queue = std::priority_queue; - llama_sp_symbol::index left; - llama_sp_symbol::index right; - float score; - size_t size; -}; - -// original implementation: -// https://github.com/ggerganov/llama.cpp/commit/074bea2eb1f1349a0118239c4152914aecaa1be4 -struct llama_tokenizer { - llama_tokenizer(const llama_vocab & vocab): vocab_(vocab) {} - - void tokenize(const std::string & text, std::vector & output) { - // split string into utf8 chars - int index = 0; - size_t offs = 0; - while (offs < text.size()) { - llama_sp_symbol sym; - size_t char_len = std::min(text.size() - offs, utf8_len(text[offs])); - sym.text = text.c_str() + offs; - sym.n = char_len; - offs += char_len; - sym.prev = index - 1; - sym.next = offs == text.size() ? -1 : index + 1; - index++; - symbols_.emplace_back(sym); - } - - // seed the work queue with all possible 2-character tokens. - for (size_t i = 1; i < symbols_.size(); ++i) { - try_add_bigram(i - 1, i); - } - - // keep substituting the highest frequency pairs for as long as we can. - while (!work_queue_.empty()) { - auto bigram = work_queue_.top(); - work_queue_.pop(); - - auto & left_sym = symbols_[bigram.left]; - auto & right_sym = symbols_[bigram.right]; - - // if one of the symbols already got merged, skip it. - if (left_sym.n == 0 || right_sym.n == 0 || - left_sym.n + right_sym.n != bigram.size) { - continue; - } - - // merge the right sym into the left one - left_sym.n += right_sym.n; - right_sym.n = 0; - - //printf("left = '%*s' size = %zu\n", (int) left_sym.n, left_sym.text, bigram.size); - - // remove the right sym from the chain - left_sym.next = right_sym.next; - if (right_sym.next >= 0) { - symbols_[right_sym.next].prev = bigram.left; - } - - // find more substitutions - try_add_bigram(left_sym.prev, bigram.left); - try_add_bigram(bigram.left, left_sym.next); - } - - for (int i = 0; i != -1; i = symbols_[i].next) { - auto & symbol = symbols_[i]; - auto token = vocab_.token_to_id.find(std::string(symbol.text, symbol.n)); - - if (token == vocab_.token_to_id.end()) { - // output any symbols that did not form tokens as bytes. - for (int j = 0; j < (int) symbol.n; ++j) { - llama_vocab::id token_id = static_cast(symbol.text[j]) + 3; - output.push_back(token_id); - } - } else { - output.push_back((*token).second); - } - } - } - -private: - void try_add_bigram(int left, int right) { - if (left == -1 || right == -1) { - return; - } - - const std::string text = std::string(symbols_[left].text, symbols_[left].n + symbols_[right].n); - auto token = vocab_.token_to_id.find(text); - - if (token == vocab_.token_to_id.end()) { - return; - } - - if (static_cast((*token).second) >= vocab_.id_to_token.size()) { - return; - } - - const auto &tok_score = vocab_.id_to_token[(*token).second]; - - llama_sp_bigram bigram; - bigram.left = left; - bigram.right = right; - bigram.score = tok_score.score; - bigram.size = text.size(); - work_queue_.push(bigram); - } - - const llama_vocab & vocab_; - std::vector symbols_; - llama_sp_bigram::queue work_queue_; -}; - -static std::vector llama_tokenize(const llama_vocab & vocab, const std::string & text, bool bos) { - llama_tokenizer tokenizer(vocab); - std::vector output; - - if (text.empty()) { - return output; - } - - if (bos) { - output.push_back(llama_token_bos()); - } - - tokenizer.tokenize(text, output); - return output; -} - -// -// grammar - internal -// - -struct llama_grammar { - const std::vector> rules; - std::vector> stacks; -}; - -struct llama_grammar_candidate { - size_t index; - const uint32_t * code_points; -}; - -// NOTE: assumes valid utf8 (but checks for overrun) -// adds a terminating 0 for use as pointer -std::vector decode_utf8(const char * src) { - static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; - const char * pos = src; - std::vector code_points; - while (*pos != 0) { - uint8_t first_byte = static_cast(*pos); - uint8_t highbits = first_byte >> 4; - int len = lookup[highbits]; - uint8_t mask = (1 << (8 - len)) - 1; - uint32_t value = first_byte & mask; - const char * end = pos + len; // may overrun! - ++pos; - for ( ; pos < end && *pos != 0; ++pos) { - value = (value << 6) + (static_cast(*pos) & 0x3F); - } - code_points.push_back(value); - } - code_points.push_back(0); - return code_points; -} - -// returns true iff pos points to the end of one of the definitions of a rule -static bool llama_grammar_is_end_of_sequence(const llama_grammar_element * pos) { - switch (pos->type) { - case LLAMA_GRETYPE_END: return true; - case LLAMA_GRETYPE_ALT: return true; - default: return false; - } -} - -// returns true iff chr satisfies the char range at pos (regular or inverse range) -// asserts that pos is pointing to a char range element -static std::pair llama_grammar_match_char( - const llama_grammar_element * pos, - const uint32_t chr) { - - bool found = false; - bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR; - GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT); - - do { - if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) { - // inclusive range, e.g. [a-z] - found = found || (pos->value <= chr && chr <= pos[1].value); - pos += 2; - } else { - // exact char match, e.g. [a] or "a" - found = found || pos->value == chr; - pos += 1; - } - } while (pos->type == LLAMA_GRETYPE_CHAR_ALT); - - return std::make_pair(found == is_positive_char, pos); -} - -// transforms a grammar pushdown stack into N possible stacks, all ending -// at a character range (terminal element) -static void llama_grammar_advance_stack( - const std::vector> & rules, - const std::vector & stack, - std::vector> & new_stacks) { - - if (stack.empty()) { - new_stacks.push_back(stack); - return; - } - - const llama_grammar_element * pos = stack.back(); - - switch (pos->type) { - case LLAMA_GRETYPE_RULE_REF: { - const size_t rule_id = static_cast(pos->value); - const llama_grammar_element * subpos = rules[rule_id].data(); - do { - // init new stack without the top (pos) - std::vector new_stack(stack.begin(), stack.end() - 1); - if (!llama_grammar_is_end_of_sequence(pos + 1)) { - // if this rule ref is followed by another element, add that to stack - new_stack.push_back(pos + 1); - } - if (!llama_grammar_is_end_of_sequence(subpos)) { - // if alternate is nonempty, add to stack - new_stack.push_back(subpos); - } - llama_grammar_advance_stack(rules, new_stack, new_stacks); - while (!llama_grammar_is_end_of_sequence(subpos)) { - // scan to end of alternate def - subpos++; - } - if (subpos->type == LLAMA_GRETYPE_ALT) { - // there's another alternate def of this rule to process - subpos++; - } else { - break; - } - } while (true); - break; - } - case LLAMA_GRETYPE_CHAR: - case LLAMA_GRETYPE_CHAR_NOT: - new_stacks.push_back(stack); - break; - default: - // end of alternate (LLAMA_GRETYPE_END, LLAMA_GRETYPE_ALT) or middle of char range - // (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER); stack should never be left on - // those - GGML_ASSERT(false); - } -} - -// takes a set of possible pushdown stacks on a grammar, which are required to -// be positioned at a character range (see `llama_grammar_advance_stack`), and -// produces the N possible stacks if the given char is accepted at those -// positions -static std::vector> llama_grammar_accept( - const std::vector> & rules, - const std::vector> & stacks, - const uint32_t chr) { - - std::vector> new_stacks; - - for (const auto & stack : stacks) { - if (stack.empty()) { - continue; - } - - auto match = llama_grammar_match_char(stack.back(), chr); - if (match.first) { - const llama_grammar_element * pos = match.second; - - // update top of stack to next element, if any - std::vector new_stack(stack.begin(), stack.end() - 1); - if (!llama_grammar_is_end_of_sequence(pos)) { - new_stack.push_back(pos); - } - llama_grammar_advance_stack(rules, new_stack, new_stacks); - } - } - - return new_stacks; -} - -static std::vector llama_grammar_reject_candidates( - const std::vector> & rules, - const std::vector> & stacks, - const std::vector & candidates); - -static std::vector llama_grammar_reject_candidates_for_stack( - const std::vector> & rules, - const std::vector & stack, - const std::vector & candidates) { - - std::vector rejects; - - if (stack.empty()) { - // accept nothing; EOS is handled elsewhere - rejects.insert(rejects.end(), candidates.begin(), candidates.end()); - return rejects; - } - - const llama_grammar_element * stack_pos = stack.back(); - - std::vector next_candidates; - for (auto tok : candidates) { - if (llama_grammar_match_char(stack_pos, tok.code_points[0]).first) { - if (tok.code_points[1] != 0) { - next_candidates.push_back({ tok.index, tok.code_points + 1 }); - } - } else { - rejects.push_back(tok); - } - } - - auto stack_pos_after = llama_grammar_match_char(stack_pos, 0).second; - - // update top of stack to next element, if any - std::vector stack_after(stack.begin(), stack.end() - 1); - if (!llama_grammar_is_end_of_sequence(stack_pos_after)) { - stack_after.push_back(stack_pos_after); - } - std::vector> next_stacks; - llama_grammar_advance_stack(rules, stack_after, next_stacks); - - auto next_rejects = llama_grammar_reject_candidates(rules, next_stacks, next_candidates); - for (auto tok : next_rejects) { - rejects.push_back({ tok.index, tok.code_points - 1 }); - } - - return rejects; -} - -static std::vector llama_grammar_reject_candidates( - const std::vector> & rules, - const std::vector> & stacks, - const std::vector & candidates) { - GGML_ASSERT(!stacks.empty()); // REVIEW - - if (candidates.empty()) { - return std::vector(); - } - - auto rejects = llama_grammar_reject_candidates_for_stack(rules, stacks.front(), candidates); - - for (size_t i = 1, size = stacks.size(); i < size; ++i) { - rejects = llama_grammar_reject_candidates_for_stack(rules, stacks[i], rejects); - } - return rejects; -} - -// -// grammar - external -// - -struct llama_grammar * llama_grammar_init( - const llama_grammar_element ** rules, - size_t n_rules, - size_t start_rule_index) { - const llama_grammar_element * pos; - - // copy rule definitions into vectors - std::vector> vec_rules(n_rules); - for (size_t i = 0; i < n_rules; i++) { - for (pos = rules[i]; pos->type != LLAMA_GRETYPE_END; pos++) { - vec_rules[i].push_back(*pos); - } - vec_rules[i].push_back({LLAMA_GRETYPE_END, 0}); - } - - // loop over alternates of start rule to build initial stacks - std::vector> stacks; - pos = rules[start_rule_index]; - do { - std::vector stack; - if (!llama_grammar_is_end_of_sequence(pos)) { - // if alternate is nonempty, add to stack - stack.push_back(pos); - } - llama_grammar_advance_stack(vec_rules, stack, stacks); - while (!llama_grammar_is_end_of_sequence(pos)) { - // scan to end of alternate def - pos++; - } - if (pos->type == LLAMA_GRETYPE_ALT) { - // there's another alternate def of this rule to process - pos++; - } else { - break; - } - } while (true); - - return new llama_grammar{ std::move(vec_rules), std::move(stacks) }; -} - -void llama_grammar_free(struct llama_grammar * grammar) { - delete grammar; -} - -// -// sampling -// - -void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates) { - assert(candidates->size > 0); - - const int64_t t_start_sample_us = ggml_time_us(); - - // Sort the logits in descending order - if (!candidates->sorted) { - std::sort(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) { - return a.logit > b.logit; - }); - candidates->sorted = true; - } - - float max_l = candidates->data[0].logit; - float cum_sum = 0.0f; - for (size_t i = 0; i < candidates->size; ++i) { - float p = expf(candidates->data[i].logit - max_l); - candidates->data[i].p = p; - cum_sum += p; - } - for (size_t i = 0; i < candidates->size; ++i) { - candidates->data[i].p /= cum_sum; - } - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } -} - -void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep) { - const int64_t t_start_sample_us = ggml_time_us(); - - k = std::max(k, (int) min_keep); - k = std::min(k, (int) candidates->size); - - // Sort scores in descending order - if (!candidates->sorted) { - auto comp = [](const llama_token_data & a, const llama_token_data & b) { - return a.logit > b.logit; - }; - if (k == (int) candidates->size) { - std::sort(candidates->data, candidates->data + candidates->size, comp); - } else { - std::partial_sort(candidates->data, candidates->data + k, candidates->data + candidates->size, comp); - } - candidates->sorted = true; - } - candidates->size = k; - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } -} - -void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) { - if (p >= 1.0f) { - return; - } - - llama_sample_softmax(ctx, candidates); - - const int64_t t_start_sample_us = ggml_time_us(); - - // Compute the cumulative probabilities - float cum_sum = 0.0f; - size_t last_idx = candidates->size; - - for (size_t i = 0; i < candidates->size; ++i) { - cum_sum += candidates->data[i].p; - - // Check if the running sum is at least p or if we have kept at least min_keep tokens - // we set the last index to i+1 to indicate that the current iterate should be included in the set - if (cum_sum >= p && i + 1 >= min_keep) { - last_idx = i + 1; - break; - } - } - - // Resize the output vector to keep only the top-p tokens - candidates->size = last_idx; - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } -} - -void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep) { - if (z >= 1.0f || candidates->size <= 2) { - return; - } - - llama_sample_softmax(nullptr, candidates); - const int64_t t_start_sample_us = ggml_time_us(); - - // Compute the first and second derivatives - std::vector first_derivatives(candidates->size - 1); - std::vector second_derivatives(candidates->size - 2); - - for (size_t i = 0; i < first_derivatives.size(); ++i) { - first_derivatives[i] = candidates->data[i].p - candidates->data[i + 1].p; - } - for (size_t i = 0; i < second_derivatives.size(); ++i) { - second_derivatives[i] = first_derivatives[i] - first_derivatives[i + 1]; - } - - // Calculate absolute value of second derivatives - for (size_t i = 0; i < second_derivatives.size(); ++i) { - second_derivatives[i] = abs(second_derivatives[i]); - } - - // Normalize the second derivatives - { - const float second_derivatives_sum = std::accumulate(second_derivatives.begin(), second_derivatives.end(), 0.0f); - - if (second_derivatives_sum > 1e-6f) { - for (float & value : second_derivatives) { - value /= second_derivatives_sum; - } - } else { - for (float & value : second_derivatives) { - value = 1.0f / second_derivatives.size(); - } - } - } - - float cum_sum = 0.0f; - size_t last_idx = candidates->size; - for (size_t i = 0; i < second_derivatives.size(); ++i) { - cum_sum += second_derivatives[i]; - - // Check if the running sum is greater than z or if we have kept at least min_keep tokens - if (cum_sum > z && i >= min_keep) { - last_idx = i; - break; - } - } - - // Resize the output vector to keep only the tokens above the tail location - candidates->size = last_idx; - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } -} - - -void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) { - // Reference implementation: - // https://github.com/huggingface/transformers/compare/main...cimeister:typical-sampling:typical-pr - if (p >= 1.0f) { - return; - } - - // Compute the softmax of logits and calculate entropy - llama_sample_softmax(nullptr, candidates); - - const int64_t t_start_sample_us = ggml_time_us(); - - float entropy = 0.0f; - for (size_t i = 0; i < candidates->size; ++i) { - entropy += -candidates->data[i].p * logf(candidates->data[i].p); - } - - // Compute the absolute difference between negative log probability and entropy for each candidate - std::vector shifted_scores; - for (size_t i = 0; i < candidates->size; ++i) { - float shifted_score = fabsf(-logf(candidates->data[i].p) - entropy); - shifted_scores.push_back(shifted_score); - } - - // Sort tokens based on the shifted_scores and their corresponding indices - std::vector indices(candidates->size); - std::iota(indices.begin(), indices.end(), 0); - - std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) { - return shifted_scores[a] < shifted_scores[b]; - }); - - // Compute the cumulative probabilities - float cum_sum = 0.0f; - size_t last_idx = indices.size(); - - for (size_t i = 0; i < indices.size(); ++i) { - size_t idx = indices[i]; - cum_sum += candidates->data[idx].p; - - // Check if the running sum is greater than typical or if we have kept at least min_keep tokens - if (cum_sum > p && i >= min_keep - 1) { - last_idx = i + 1; - break; - } - } - - // Resize the output vector to keep only the locally typical tokens - std::vector new_candidates; - for (size_t i = 0; i < last_idx; ++i) { - size_t idx = indices[i]; - new_candidates.push_back(candidates->data[idx]); - } - - // Replace the data in candidates with the new_candidates data - std::copy(new_candidates.begin(), new_candidates.end(), candidates->data); - candidates->size = new_candidates.size(); - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } -} - -void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates_p, float temp) { - const int64_t t_start_sample_us = ggml_time_us(); - - for (size_t i = 0; i < candidates_p->size; ++i) { - candidates_p->data[i].logit /= temp; - } - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } -} - -void llama_sample_repetition_penalty(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float penalty) { - if (last_tokens_size == 0 || penalty == 1.0f) { - return; - } - - const int64_t t_start_sample_us = ggml_time_us(); - - for (size_t i = 0; i < candidates->size; ++i) { - const auto * token_iter = std::find(last_tokens, last_tokens + last_tokens_size, candidates->data[i].id); - if (token_iter == last_tokens + last_tokens_size) { - continue; - } - - // The academic publication that described this technique actually just only divided, but that would cause tokens with negative logits to become more likely, which is obviously wrong. - // This is common fix for this problem, which is to multiply by the penalty instead of dividing. - if (candidates->data[i].logit <= 0) { - candidates->data[i].logit *= penalty; - } else { - candidates->data[i].logit /= penalty; - } - } - - candidates->sorted = false; - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } -} - -void llama_sample_frequency_and_presence_penalties(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens_p, size_t last_tokens_size, float alpha_frequency, float alpha_presence) { - if (last_tokens_size == 0 || (alpha_frequency == 0.0f && alpha_presence == 0.0f)) { - return; - } - - const int64_t t_start_sample_us = ggml_time_us(); - - // Create a frequency map to count occurrences of each token in last_tokens - std::unordered_map token_count; - for (size_t i = 0; i < last_tokens_size; ++i) { - token_count[last_tokens_p[i]]++; - } - - // Apply frequency and presence penalties to the candidates - for (size_t i = 0; i < candidates->size; ++i) { - auto token_iter = token_count.find(candidates->data[i].id); - if (token_iter == token_count.end()) { - continue; - } - - int count = token_iter->second; - candidates->data[i].logit -= float(count) * alpha_frequency + float(count > 0) * alpha_presence; - } - - candidates->sorted = false; - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } -} - -void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * candidates, const struct llama_grammar * grammar) { - assert(ctx); - const int64_t t_start_sample_us = ggml_time_us(); - - bool allow_eos = false; - for (const auto & stack : grammar->stacks) { - if (stack.empty()) { - allow_eos = true; - break; - } - } - - const llama_token eos = llama_token_eos(); - - std::vector> candidates_decoded; - std::vector candidates_grammar; - - for (size_t i = 0; i < candidates->size; ++i) { - const llama_token id = candidates->data[i].id; - const char * str = llama_token_to_str(ctx, id); - if (id == eos) { - if (!allow_eos) { - candidates->data[i].logit = -INFINITY; - } - } else if (*str == 0) { - candidates->data[i].logit = -INFINITY; - } else { - candidates_decoded.push_back(decode_utf8(str)); - candidates_grammar.push_back({ i, candidates_decoded.back().data() }); - } - } - - const auto rejects = - llama_grammar_reject_candidates(grammar->rules, grammar->stacks, candidates_grammar); - for (auto & reject : rejects) { - candidates->data[reject.index].logit = -INFINITY; - } - - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; -} - -static void llama_log_softmax(float * array, size_t size) { - float max_l = *std::max_element(array, array + size); - float sum = 0.f; - for (size_t i = 0; i < size; ++i) { - float p = expf(array[i] - max_l); - sum += p; - array[i] = p; - } - - for (size_t i = 0; i < size; ++i) { - array[i] = logf(array[i] / sum); - } -} - -void llama_sample_classifier_free_guidance( - struct llama_context * ctx, - llama_token_data_array * candidates, - struct llama_context * guidance_ctx, - float scale) { - int64_t t_start_sample_us = ggml_time_us(); - - assert(ctx); - auto n_vocab = llama_n_vocab(ctx); - assert(n_vocab == (int)candidates->size); - assert(!candidates->sorted); - - std::vector logits_base; - logits_base.reserve(candidates->size); - for (size_t i = 0; i < candidates->size; ++i) { - logits_base.push_back(candidates->data[i].logit); - } - llama_log_softmax(logits_base.data(), candidates->size); - - float* logits_guidance = llama_get_logits(guidance_ctx); - llama_log_softmax(logits_guidance, n_vocab); - - for (int i = 0; i < n_vocab; ++i) { - float logit_guidance = logits_guidance[i]; - float logit_base = logits_base[i]; - candidates->data[i].logit = scale * (logit_base - logit_guidance) + logit_guidance; - } - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } -} - -llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int m, float * mu) { - assert(ctx); - auto N = float(llama_n_vocab(ctx)); - int64_t t_start_sample_us; - t_start_sample_us = ggml_time_us(); - - llama_sample_softmax(nullptr, candidates); - - // Estimate s_hat using the most probable m tokens - float s_hat = 0.0; - float sum_ti_bi = 0.0; - float sum_ti_sq = 0.0; - for (size_t i = 0; i < size_t(m - 1) && i < candidates->size - 1; ++i) { - float t_i = logf(float(i + 2) / float(i + 1)); - float b_i = logf(candidates->data[i].p / candidates->data[i + 1].p); - sum_ti_bi += t_i * b_i; - sum_ti_sq += t_i * t_i; - } - s_hat = sum_ti_bi / sum_ti_sq; - - // Compute k from the estimated s_hat and target surprise value - float epsilon_hat = s_hat - 1; - float k = powf((epsilon_hat * powf(2, *mu)) / (1 - powf(N, -epsilon_hat)), 1 / s_hat); - - // Sample the next word X using top-k sampling - llama_sample_top_k(nullptr, candidates, int(k), 1); - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } - llama_token X = llama_sample_token(ctx, candidates); - t_start_sample_us = ggml_time_us(); - - // Compute error as the difference between observed surprise and target surprise value - size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) { - return candidate.id == X; - })); - float observed_surprise = -log2f(candidates->data[X_idx].p); - float e = observed_surprise - tau; - - // Update mu using the learning rate and error - *mu = *mu - eta * e; - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } - return X; -} - -llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu) { - int64_t t_start_sample_us; - t_start_sample_us = ggml_time_us(); - - llama_sample_softmax(ctx, candidates); - - // Truncate the words with surprise values greater than mu - candidates->size = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) { - return -log2f(candidate.p) > *mu; - })); - - if (candidates->size == 0) { - candidates->size = 1; - } - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } - - // Normalize the probabilities of the remaining words - llama_sample_softmax(ctx, candidates); - - // Sample the next word X from the remaining words - llama_token X = llama_sample_token(ctx, candidates); - t_start_sample_us = ggml_time_us(); - - // Compute error as the difference between observed surprise and target surprise value - size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) { - return candidate.id == X; - })); - float observed_surprise = -log2f(candidates->data[X_idx].p); - float e = observed_surprise - tau; - - // Update mu using the learning rate and error - *mu = *mu - eta * e; - - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } - return X; -} - -llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates) { - const int64_t t_start_sample_us = ggml_time_us(); - - // Find max element - auto * max_iter = std::max_element(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) { - return a.logit < b.logit; - }); - - llama_token result = max_iter->id; - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - ctx->n_sample++; - } - return result; -} - -llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates) { - assert(ctx); - const int64_t t_start_sample_us = ggml_time_us(); - llama_sample_softmax(nullptr, candidates); - - std::vector probs; - probs.reserve(candidates->size); - for (size_t i = 0; i < candidates->size; ++i) { - probs.push_back(candidates->data[i].p); - } - - std::discrete_distribution<> dist(probs.begin(), probs.end()); - auto & rng = ctx->rng; - int idx = dist(rng); - - llama_token result = candidates->data[idx].id; - - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - ctx->n_sample++; - return result; -} - -void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar * grammar, llama_token token) { - const int64_t t_start_sample_us = ggml_time_us(); - - if (token == llama_token_eos()) { - for (const auto & stack : grammar->stacks) { - if (stack.empty()) { - return; - } - } - GGML_ASSERT(false); - } - - const char * str = llama_token_to_str(ctx, token); - // Note terminating 0 in decoded string - auto code_points = decode_utf8(str); - for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { - grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); - } - GGML_ASSERT(!grammar->stacks.empty()); - - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; -} - -// -// quantization -// - -static void llama_convert_tensor_internal(const gguf_load_tensor & tensor, gguf_buffer & output, const int nelements, const int nthread) { - if (output.size < nelements * sizeof(float)) { - output.resize(nelements * sizeof(float)); - } - float * f32_output = (float *) output.addr; - - ggml_type_traits_t qtype; - if (ggml_is_quantized(tensor.type)) { - qtype = ggml_internal_get_type_traits(tensor.type); - if (qtype.to_float == NULL) { - throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor.type))); - } - } else if (tensor.type != GGML_TYPE_F16) { - throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor.type))); - } - - if (nthread < 2) { - if (tensor.type == GGML_TYPE_F16) { - ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor.data, f32_output, nelements); - } else if (ggml_is_quantized(tensor.type)) { - qtype.to_float(tensor.data, f32_output, nelements); - } else { - GGML_ASSERT(false); // unreachable - } - return; - } - - auto block_size = tensor.type == GGML_TYPE_F16 ? 1 : (size_t)ggml_blck_size(tensor.type); - auto block_size_bytes = ggml_type_size(tensor.type); - - GGML_ASSERT(nelements % block_size == 0); - auto nblocks = nelements / block_size; - auto blocks_per_thread = nblocks / nthread; - auto spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count - - std::vector workers; - for (auto tnum = 0, in_buff_offs = 0, out_buff_offs = 0; tnum < nthread; tnum++) { - auto thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread - auto thr_elems = thr_blocks * block_size; // number of elements for this thread - auto thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread - - auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) { - if (typ == GGML_TYPE_F16) { - ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels); - } else { - qtype.to_float(inbuf, outbuf, nels); - } - }; - workers.push_back(std::thread(compute, tensor.type, tensor.data + in_buff_offs, f32_output + out_buff_offs, thr_elems)); - in_buff_offs += thr_block_bytes; - out_buff_offs += thr_elems; - } - for (auto & worker : workers) { - worker.join(); - } - -} - -static void llama_model_quantize_internal(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) { - ggml_type quantized_type; - llama_ftype ftype = params->ftype; - int nthread = params->nthread; - - switch (params->ftype) { - case LLAMA_FTYPE_MOSTLY_Q4_0: quantized_type = GGML_TYPE_Q4_0; break; - case LLAMA_FTYPE_MOSTLY_Q4_1: quantized_type = GGML_TYPE_Q4_1; break; - case LLAMA_FTYPE_MOSTLY_Q5_0: quantized_type = GGML_TYPE_Q5_0; break; - case LLAMA_FTYPE_MOSTLY_Q5_1: quantized_type = GGML_TYPE_Q5_1; break; - case LLAMA_FTYPE_MOSTLY_Q8_0: quantized_type = GGML_TYPE_Q8_0; break; - case LLAMA_FTYPE_MOSTLY_F16: quantized_type = GGML_TYPE_F16; break; - case LLAMA_FTYPE_ALL_F32: quantized_type = GGML_TYPE_F32; break; - -#ifdef GGML_USE_K_QUANTS - // K-quants - case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break; - case LLAMA_FTYPE_MOSTLY_Q3_K_S: - case LLAMA_FTYPE_MOSTLY_Q3_K_M: - case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break; - case LLAMA_FTYPE_MOSTLY_Q4_K_S: - case LLAMA_FTYPE_MOSTLY_Q4_K_M: quantized_type = GGML_TYPE_Q4_K; break; - case LLAMA_FTYPE_MOSTLY_Q5_K_S: - case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break; - case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break; -#endif - default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); - } - - if (nthread <= 0) { - nthread = std::thread::hardware_concurrency(); - } - - std::unique_ptr model_loader(new llama_model_loader(fname_inp, /*use_mmap*/ false)); - gguf_file_saver file_saver(fname_out.c_str(), model_loader->file_loader.get(), params->ftype); - -#ifdef GGML_USE_K_QUANTS - int n_attention_wv = 0; - int n_feed_forward_w2 = 0; - for (auto& tensor : model_loader->tensors_map.tensors) { - if (tensor.name.find("attention.wv.weight") != std::string::npos) { - ++n_attention_wv; - } - else if (tensor.name.find("feed_forward.w2.weight") != std::string::npos) { - ++n_feed_forward_w2; - } - } - - int i_attention_wv = 0; - int i_feed_forward_w2 = 0; -#endif - - size_t total_size_org = 0; - size_t total_size_new = 0; - std::vector hist_all(1 << 4, 0); - - std::vector workers; - std::mutex mutex; - - auto use_more_bits = [] (int i_layer, int num_layers) -> bool { - return i_layer < num_layers/8 || i_layer >= 7*num_layers/8 || (i_layer - num_layers/8)%3 == 2; - }; - - size_t idx = 0; - for (gguf_load_tensor & tensor : model_loader->tensors_map.tensors) { - gguf_buffer read_data; - read_data.resize(tensor.size); - tensor.data = read_data.addr; - model_loader->load_data_for(tensor); - - printf("[%4zu/%4zu] %36s - %16s, type = %6s, ", - ++idx, model_loader->tensors_map.tensors.size(), - tensor.name.c_str(), llama_format_tensor_shape(tensor.ne).c_str(), - ggml_type_name(tensor.type)); - - // This used to be a regex, but has an extreme cost to compile times. - bool quantize = tensor.name.rfind("weight") == tensor.name.size() - 6; // ends with 'weight'? - - // quantize only 2D tensors - quantize &= (tensor.ne.size() == 2); - quantize &= params->quantize_output_tensor || tensor.name != "output.weight"; - quantize &= quantized_type != tensor.type; - - enum ggml_type new_type; - void * new_data; - size_t new_size; - gguf_buffer work; - - if (!quantize) { - new_type = tensor.type; - new_data = tensor.data; - new_size = tensor.size; - printf("size = %8.3f MB\n", tensor.size/1024.0/1024.0); - } else { - new_type = quantized_type; -#ifdef GGML_USE_K_QUANTS - if (tensor.name == "output.weight") { - int nx = tensor.ne.at(0); - int ny = tensor.ne.at(1); - if (nx % QK_K == 0 && ny % QK_K == 0) { - new_type = GGML_TYPE_Q6_K; - } - } else if (tensor.name.find("attention.wv.weight") != std::string::npos) { - if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; - else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) && - use_more_bits(i_attention_wv, n_attention_wv)) new_type = GGML_TYPE_Q6_K; - else if (QK_K == 64 && (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S) && - (i_attention_wv < n_attention_wv/8 || i_attention_wv >= 7*n_attention_wv/8)) new_type = GGML_TYPE_Q6_K; - ++i_attention_wv; - } else if (tensor.name.find("feed_forward.w2.weight") != std::string::npos) { - if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; - else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) && - use_more_bits(i_feed_forward_w2, n_feed_forward_w2)) new_type = GGML_TYPE_Q6_K; - //else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && i_feed_forward_w2 < n_feed_forward_w2/8) new_type = GGML_TYPE_Q6_K; - ++i_feed_forward_w2; - } else if (tensor.name.find("attention.wo.weight") != std::string::npos) { - if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; - } - bool convert_incompatible_tensor = false; - if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K || - new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K) { - int nx = tensor.ne.at(0); - int ny = tensor.ne.at(1); - if (nx % QK_K != 0 || ny % QK_K != 0) { - fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K); - convert_incompatible_tensor = true; - } - } - if (convert_incompatible_tensor) { - if (tensor.name == "output.weight") { - new_type = GGML_TYPE_F16; //fall back to F16 instead of just failing. - fprintf(stderr, "F16 will be used for this tensor instead.\n"); - } else if (tensor.name == "tok_embeddings.weight") { - new_type = GGML_TYPE_Q4_0; //fall back to Q4_0 instead of just failing. - fprintf(stderr, "Q4_0 will be used for this tensor instead.\n"); - } else { - throw std::runtime_error("Unsupported tensor size encountered\n"); - } - } -#endif - - float * f32_data; - size_t nelements = tensor.ne.at(0) * tensor.ne.at(1); - gguf_buffer f32_conv_buf; - - if (tensor.type == GGML_TYPE_F32) { - f32_data = (float *) tensor.data; - } else if (ggml_is_quantized(tensor.type) && !params->allow_requantize) { - throw std::runtime_error(format("requantizing from type %s is disabled", ggml_type_name(tensor.type))); - } else { - llama_convert_tensor_internal(tensor, f32_conv_buf, nelements, nthread); - f32_data = (float *) f32_conv_buf.addr; - } - - printf("quantizing to %s .. ", ggml_type_name(new_type)); - fflush(stdout); - - work.resize(nelements * 4); // upper bound on size - new_data = work.addr; - std::vector hist_cur(1 << 4, 0); - - int chunk_size = 32 * 512; - const int nchunk = (nelements + chunk_size - 1)/chunk_size; - const int nthread_use = nthread > 1 ? std::max(1, std::min(nthread, nchunk)) : 1; - if (nthread_use < 2) { - new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nelements, hist_cur.data()); - } else { - size_t counter = 0; - new_size = 0; - auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, nelements, chunk_size] () { - std::vector local_hist; - size_t local_size = 0; - while (true) { - std::unique_lock lock(mutex); - size_t first = counter; counter += chunk_size; - if (first >= nelements) { - if (!local_hist.empty()) { - for (int j=0; j %8.2f MB | hist: ", tensor.size/1024.0/1024.0, new_size/1024.0/1024.0); - int64_t tot_count = 0; - for (size_t i = 0; i < hist_cur.size(); i++) { - hist_all[i] += hist_cur[i]; - tot_count += hist_cur[i]; - } - - if (tot_count > 0) { - for (size_t i = 0; i < hist_cur.size(); i++) { - printf("%5.3f ", hist_cur[i] / float(nelements)); - } - } - printf("\n"); - } - total_size_org += tensor.size; - total_size_new += new_size; - file_saver.write_tensor(tensor, new_type, new_data, new_size); - } - - printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0); - printf("%s: quant size = %8.2f MB\n", __func__, total_size_new/1024.0/1024.0); - - { - int64_t sum_all = 0; - for (size_t i = 0; i < hist_all.size(); i++) { - sum_all += hist_all[i]; - } - - if (sum_all > 0) { - printf("%s: hist: ", __func__); - for (size_t i = 0; i < hist_all.size(); i++) { - printf("%5.3f ", hist_all[i] / float(sum_all)); - } - printf("\n"); - } - } -} - - - -// -// interface implementation -// - -struct llama_model * llama_load_model_from_file( - const char * path_model, - struct llama_context_params params) { - ggml_time_init(); - - llama_model * model = new llama_model; - - ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32; - - if (!llama_model_load(path_model, *model, model->vocab, params.n_ctx, params.n_batch, params.n_gqa, params.rms_norm_eps, params.n_gpu_layers, - params.main_gpu, params.tensor_split, params.rope_freq_base, params.rope_freq_scale,params.low_vram, - memory_type, params.use_mmap, params.use_mlock, params.vocab_only, params.progress_callback, - params.progress_callback_user_data)) { - delete model; - fprintf(stderr, "%s: failed to load model\n", __func__); - return nullptr; - } - - return model; -} - -void llama_free_model(struct llama_model * model) { - delete model; -} - -struct llama_context * llama_new_context_with_model( - struct llama_model * model, - struct llama_context_params params) { - - if (!model) { - return nullptr; - } - - llama_context * ctx = new llama_context(*model); - - if (params.seed == LLAMA_DEFAULT_SEED) { - params.seed = time(NULL); - } - - unsigned cur_percentage = 0; - if (params.progress_callback == NULL) { - params.progress_callback_user_data = &cur_percentage; - params.progress_callback = [](float progress, void * ctx) { - unsigned * cur_percentage_p = (unsigned *) ctx; - unsigned percentage = (unsigned) (100 * progress); - while (percentage > *cur_percentage_p) { - *cur_percentage_p = percentage; - fprintf(stderr, "."); - fflush(stderr); - if (percentage >= 100) { - fprintf(stderr, "\n"); - } - } - }; - } - - ctx->rng = std::mt19937(params.seed); - ctx->logits_all = params.logits_all; - - ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32; - - // reserve memory for context buffers - if (!params.vocab_only) { - if (!kv_cache_init(ctx->model.hparams, ctx->kv_self, memory_type, ctx->model.hparams.n_ctx, params.n_gpu_layers)) { - fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__); - llama_free(ctx); - return nullptr; - } - - { - const size_t memory_size = ggml_nbytes(ctx->kv_self.k) + ggml_nbytes(ctx->kv_self.v); - fprintf(stderr, "%s: kv self size = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0); - } - - const auto & hparams = ctx->model.hparams; - - // resized during inference - if (params.logits_all) { - ctx->logits.reserve(hparams.n_ctx*hparams.n_vocab); - } else { - ctx->logits.reserve(hparams.n_vocab); - } - - if (params.embedding){ - ctx->embedding.resize(hparams.n_embd); - } - - ctx->buf_compute.resize(MEM_REQ_EVAL().at(ctx->model.type) + ggml_graph_overhead()); - - ctx->buf_scratch[0].resize(MEM_REQ_SCRATCH0(hparams.n_ctx).at(ctx->model.type)); - ctx->buf_scratch[1].resize(MEM_REQ_SCRATCH1().at(ctx->model.type)); - } - -#ifdef GGML_USE_METAL - if (params.n_gpu_layers > 0) { - // this allocates all Metal resources and memory buffers - ctx->ctx_metal = ggml_metal_init(1); - - void * data_ptr = NULL; - size_t data_size = 0; - - if (params.use_mmap) { - data_ptr = ctx->model.mapping->addr; - data_size = ctx->model.mapping->size; - } else { - data_ptr = ggml_get_mem_buffer(ctx->model.ctx); - data_size = ggml_get_mem_size (ctx->model.ctx); - } - - const size_t max_size = ggml_get_max_tensor_size(ctx->model.ctx); - - fprintf(stderr, "%s: max tensor size = %8.2f MB\n", __func__, max_size/1024.0/1024.0); - -#define LLAMA_METAL_CHECK_BUF(result) \ - if (!(result)) { \ - fprintf(stderr, "%s: failed to add buffer\n", __func__); \ - llama_free(ctx); \ - return NULL; \ - } - - LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "data", data_ptr, data_size, max_size)); - - LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "eval", ctx->buf_compute.addr, ctx->buf_compute.size, 0)); - LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "kv", ctx->kv_self.buf.addr, ctx->kv_self.buf.size, 0)); - - LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr0", ctx->buf_scratch[0].addr, ctx->buf_scratch[0].size, 0)); - LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr1", ctx->buf_scratch[1].addr, ctx->buf_scratch[1].size, 0)); -#undef LLAMA_METAL_CHECK_BUF - } -#endif - -#ifdef GGML_USE_MPI - ctx->ctx_mpi = ggml_mpi_init(); - - if (ggml_mpi_rank(ctx->ctx_mpi) > 0) { - // Enter a blocking eval loop with dummy input, letting rank=0 drive the process - const std::vector tmp(ctx->model.hparams.n_ctx, llama_token_bos()); - while (!llama_eval(ctx, tmp.data(), tmp.size(), 0, 0)) {}; - llama_backend_free(); - exit(1); - } -#endif - - return ctx; -} - -struct llama_context * llama_init_from_file( - const char * path_model, - struct llama_context_params params) { - - struct llama_model * model = llama_load_model_from_file(path_model, params); - if (!model) { - return nullptr; - } - struct llama_context * ctx = llama_new_context_with_model(model, params); - ctx->model_owner = true; - return ctx; -} - -void llama_free(struct llama_context * ctx) { - if (ctx->model_owner) { - delete &ctx->model; - } - delete ctx; -} - -int llama_model_quantize( - const char * fname_inp, - const char * fname_out, - const llama_model_quantize_params *params) { - try { - llama_model_quantize_internal(fname_inp, fname_out, params); - return 0; - } catch (const std::exception & err) { - fprintf(stderr, "%s: failed to quantize: %s\n", __func__, err.what()); - return 1; - } -} - -int llama_apply_lora_from_file_internal(const struct llama_model & model, const char * path_lora, const char * path_base_model, int n_threads) { - fprintf(stderr, "%s: applying lora adapter from '%s' - please wait ...\n", __func__, path_lora); - - const int64_t t_start_lora_us = ggml_time_us(); - - auto fin = std::ifstream(path_lora, std::ios::binary); - if (!fin) { - fprintf(stderr, "%s: failed to open '%s'\n", __func__, path_lora); - return 1; - } - - // verify magic and version - { - uint32_t magic; - fin.read((char *) &magic, sizeof(magic)); - uint32_t format_version; - fin.read((char *) &format_version, sizeof(format_version)); - - if (format_version != 1) { - fprintf(stderr, "%s: unsupported file version\n", __func__ ); - return 1; - } - } - - int32_t lora_r; - int32_t lora_alpha; - fin.read((char *) &lora_r, sizeof(lora_r)); - fin.read((char *) &lora_alpha, sizeof(lora_alpha)); - float scaling = (float)lora_alpha / (float)lora_r; - - fprintf(stderr, "%s: r = %d, alpha = %d, scaling = %.2f\n", __func__, lora_r, lora_alpha, scaling); - - - // create a temporary ggml context to store the lora tensors - // todo: calculate size from biggest possible tensor - std::vector lora_buf(1024ull * 1024ull * 1024ull); - struct ggml_init_params params; - params.mem_size = lora_buf.size(); - params.mem_buffer = lora_buf.data(); - params.no_alloc = false; - - ggml_context * lora_ctx = ggml_init(params); - std::unordered_map lora_tensors; - - // create a name -> tensor map of the model to accelerate lookups - std::unordered_map model_tensors; - for (const auto & kv: model.tensors_by_name) { - model_tensors.insert(kv); - } - - - // load base model - std::unique_ptr model_loader; - ggml_context * base_ctx = NULL; - gguf_buffer base_buf; - if (path_base_model) { - fprintf(stderr, "%s: loading base model from '%s'\n", __func__, path_base_model); - model_loader.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true)); - - size_t ctx_size; - size_t mmapped_size; - model_loader->calc_sizes(&ctx_size, &mmapped_size); - base_buf.resize(ctx_size); - - ggml_init_params base_params; - base_params.mem_size = base_buf.size; - base_params.mem_buffer = base_buf.addr; - base_params.no_alloc = model_loader->use_mmap; - - base_ctx = ggml_init(base_params); - - model_loader->ggml_ctx = base_ctx; - - // maybe this should in llama_model_loader - if (model_loader->use_mmap) { - model_loader->mapping.reset(new gguf_mmap(&model_loader->file_loader->file, /* prefetch */ 0, ggml_is_numa())); - } - } - - // read tensors and apply - bool warned = false; - int n_tensors = 0; - - std::vector work_buffer; - - while (true) { - int32_t n_dims; - int32_t length; - int32_t ftype; - - fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); - fin.read(reinterpret_cast(&length), sizeof(length)); - fin.read(reinterpret_cast(&ftype), sizeof(ftype)); - if (fin.eof()) { - break; - } - - int32_t ne[2] = { 1, 1 }; - for (int i = 0; i < n_dims; ++i) { - fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i])); - } - - std::string name; - { - char buf[1024]; - fin.read(buf, length); - name = std::string(buf, length); - } - - // check for lora suffix and get the type of tensor - const std::string lora_suffix = ".lora"; - size_t pos = name.rfind(lora_suffix); - if (pos == std::string::npos) { - fprintf(stderr, "%s: error: '%s' is not a lora tensor\n", __func__, name.c_str()); - return 1; - } - - std::string lora_type = name.substr(pos + lora_suffix.length()); - std::string base_name = name; - base_name.erase(pos); - // fprintf(stderr, "%s: %s => %s (lora type %s) ", __func__, name.c_str(),base_name.c_str(), lora_type.c_str()); - - if (model_tensors.find(base_name) == model_tensors.end()) { - fprintf(stderr, "%s: unknown tensor '%s' in lora adapter\n", __func__, name.data()); - return 1; - } - - // create ggml tensor - ggml_type wtype; - switch (ftype) { - case 0: wtype = GGML_TYPE_F32; break; - case 1: wtype = GGML_TYPE_F16; break; - default: - { - fprintf(stderr, "%s: invalid tensor data type '%d'\n", - __func__, ftype); - return false; - } - } - ggml_tensor * lora_tensor; - if (n_dims == 2) { - lora_tensor = ggml_new_tensor_2d(lora_ctx, wtype, ne[0], ne[1]); - } - else { - fprintf(stderr, "%s: unsupported tensor dimension %d\n", __func__, n_dims); - return 1; - } - ggml_set_name(lora_tensor, "lora_tensor"); - - // load tensor data - size_t offset = fin.tellg(); - size_t tensor_data_size = ggml_nbytes(lora_tensor); - offset = (offset + 31) & -32; - fin.seekg(offset); - fin.read((char*)lora_tensor->data, tensor_data_size); - - lora_tensors[name] = lora_tensor; - - // check if we have both A and B tensors and apply - if (lora_tensors.find(base_name + ".loraA") != lora_tensors.end() && - lora_tensors.find(base_name + ".loraB") != lora_tensors.end()) { - - ggml_tensor * dest_t = model_tensors[base_name]; - - offload_func_t offload_func = llama_nop; - offload_func_t offload_func_force_inplace = llama_nop; - -#ifdef GGML_USE_CUBLAS - if (dest_t->backend == GGML_BACKEND_GPU || dest_t->backend == GGML_BACKEND_GPU_SPLIT) { - if (dest_t->type != GGML_TYPE_F16) { - throw std::runtime_error(format( - "%s: error: the simultaneous use of LoRAs and GPU acceleration is only supported for f16 models", __func__)); - } - offload_func = ggml_cuda_assign_buffers; - offload_func_force_inplace = ggml_cuda_assign_buffers_force_inplace; - } -#endif // GGML_USE_CUBLAS - - ggml_tensor * base_t; - if (model_loader) { - // load from base model - if (model_loader->tensors_map.name_to_idx.find(base_name) == model_loader->tensors_map.name_to_idx.end()) { - fprintf(stderr, "%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str()); - return 1; - } - size_t idx = model_loader->tensors_map.name_to_idx[base_name]; - gguf_load_tensor & lt = model_loader->tensors_map.tensors[idx]; - base_t = model_loader->get_tensor(base_name, { (uint32_t)dest_t->ne[0], (uint32_t)dest_t->ne[1] }, GGML_BACKEND_CPU); - lt.data = (uint8_t *) lt.ggml_tensor->data; - model_loader->load_data_for(lt); - lt.ggml_tensor->data = lt.data; - } - else { - base_t = dest_t; - } - - if (ggml_is_quantized(base_t->type)) { - if (!warned) { - fprintf(stderr, "%s: warning: using a lora adapter with a quantized model may result in poor quality, " - "use a f16 or f32 base model with --lora-base\n", __func__); - warned = true; - } - } - - ggml_tensor * loraA = lora_tensors[base_name + ".loraA"]; - GGML_ASSERT(loraA->type == GGML_TYPE_F32); - ggml_set_name(loraA, "loraA"); - - ggml_tensor * loraB = lora_tensors[base_name + ".loraB"]; - GGML_ASSERT(loraB->type == GGML_TYPE_F32); - ggml_set_name(loraB, "loraB"); - - if (base_t->ne[0] != loraA->ne[1] || base_t->ne[1] != loraB->ne[1]) { - fprintf(stderr, "%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");" - " are you sure that this adapter is for this model?\n", __func__, base_t->ne[0], loraA->ne[1]); - return 1; - } - - // w = w + BA*s - ggml_tensor * BA = ggml_mul_mat(lora_ctx, loraA, loraB); - offload_func(BA); - ggml_set_name(BA, "BA"); - - if (scaling != 1.0f) { - ggml_tensor * scale_tensor = ggml_new_f32(lora_ctx, scaling); - ggml_set_name(scale_tensor, "scale_tensor"); - - BA = ggml_scale_inplace(lora_ctx, BA, scale_tensor); - offload_func(BA); - ggml_set_name(BA, "BA_scaled"); - } - - ggml_tensor * r; - if (base_t == dest_t) { - r = ggml_add_inplace(lora_ctx, dest_t, BA); - offload_func_force_inplace(r); - ggml_set_name(r, "r_add_inplace"); - } - else { - r = ggml_add(lora_ctx, base_t, BA); - offload_func(r); - ggml_set_name(r, "r_add"); - - r = ggml_cpy(lora_ctx, r, dest_t); - offload_func(r); - ggml_set_name(r, "r_cpy"); - } - - struct ggml_cgraph gf = ggml_build_forward(r); - - ggml_graph_compute_helper(work_buffer, &gf, n_threads); - - // we won't need these tensors again, reset the context to save memory - ggml_free(lora_ctx); - lora_ctx = ggml_init(params); - lora_tensors.clear(); - - n_tensors++; - if (n_tensors % 4 == 0) { - fprintf(stderr, "."); - } - } - } - - // TODO: this should be in a destructor, it will leak on failure - ggml_free(lora_ctx); - if (base_ctx) { - ggml_free(base_ctx); - } - - const int64_t t_lora_us = ggml_time_us() - t_start_lora_us; - fprintf(stderr, " done (%.2f ms)\n", t_lora_us / 1000.0); - - return 0; -} - -int llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lora, const char * path_base_model, int n_threads) { - try { - return llama_apply_lora_from_file_internal(ctx->model, path_lora, path_base_model, n_threads); - } catch (const std::exception & err) { - fprintf(stderr, "%s: failed to apply lora adapter: %s\n", __func__, err.what()); - return 1; - } -} - -int llama_model_apply_lora_from_file(const struct llama_model * model, const char * path_lora, const char * path_base_model, int n_threads) { - try { - return llama_apply_lora_from_file_internal(*model, path_lora, path_base_model, n_threads); - } catch (const std::exception & err) { - fprintf(stderr, "%s: failed to apply lora adapter: %s\n", __func__, err.what()); - return 1; - } -} - -int llama_get_kv_cache_token_count(const struct llama_context * ctx) { - return ctx->kv_self.n; -} - -#define LLAMA_MAX_RNG_STATE (64*1024) - -void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed) { - if (seed == LLAMA_DEFAULT_SEED) { - seed = time(NULL); - } - ctx->rng.seed(seed); -} - -// Returns the *maximum* size of the state -size_t llama_get_state_size(const struct llama_context * ctx) { - // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state. - // for reference, std::mt19937(1337) serializes to 6701 bytes. - const size_t s_rng_size = sizeof(size_t); - const size_t s_rng = LLAMA_MAX_RNG_STATE; - const size_t s_logits_capacity = sizeof(size_t); - const size_t s_logits_size = sizeof(size_t); - const size_t s_logits = ctx->logits.capacity() * sizeof(float); - const size_t s_embedding_size = sizeof(size_t); - const size_t s_embedding = ctx->embedding.size() * sizeof(float); - const size_t s_kv_size = sizeof(size_t); - const size_t s_kv_ntok = sizeof(int); - const size_t s_kv = ctx->kv_self.buf.size; - - const size_t s_total = ( - + s_rng_size - + s_rng - + s_logits_capacity - + s_logits_size - + s_logits - + s_embedding_size - + s_embedding - + s_kv_size - + s_kv_ntok - + s_kv - ); - - return s_total; -} - -// Copies the state to the specified destination address -size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { - uint8_t * out = dst; - - // copy rng - { - std::stringstream rng_ss; - rng_ss << ctx->rng; - - const size_t rng_size = rng_ss.str().size(); - char rng_buf[LLAMA_MAX_RNG_STATE]; - - memset(&rng_buf[0], 0, LLAMA_MAX_RNG_STATE); - memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size()); - - memcpy(out, &rng_size, sizeof(rng_size)); out += sizeof(rng_size); - memcpy(out, &rng_buf[0], LLAMA_MAX_RNG_STATE); out += LLAMA_MAX_RNG_STATE; - } - - // copy logits - { - const size_t logits_cap = ctx->logits.capacity(); - const size_t logits_size = ctx->logits.size(); - - memcpy(out, &logits_cap, sizeof(logits_cap)); out += sizeof(logits_cap); - memcpy(out, &logits_size, sizeof(logits_size)); out += sizeof(logits_size); - - if (logits_size) { - memcpy(out, ctx->logits.data(), logits_size * sizeof(float)); - } - - out += logits_cap * sizeof(float); - } - - // copy embeddings - { - const size_t embedding_size = ctx->embedding.size(); - - memcpy(out, &embedding_size, sizeof(embedding_size)); out += sizeof(embedding_size); - - if (embedding_size) { - memcpy(out, ctx->embedding.data(), embedding_size * sizeof(float)); - out += embedding_size * sizeof(float); - } - } - - // copy kv cache - { - const auto & kv_self = ctx->kv_self; - const auto & hparams = ctx->model.hparams; - const int n_layer = hparams.n_layer; - const int n_embd = hparams.n_embd; - const int n_ctx = hparams.n_ctx; - - const size_t kv_size = kv_self.buf.size; - const int kv_ntok = llama_get_kv_cache_token_count(ctx); - - memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size); - memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok); - - if (kv_size) { - const size_t elt_size = ggml_element_size(kv_self.k); - - ggml_context * cpy_ctx = ggml_init({ 4096, NULL, /* no_alloc */ true }); - ggml_cgraph gf{}; - - ggml_tensor * kout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_ntok, n_layer); - kout3d->data = out; - out += ggml_nbytes(kout3d); - - ggml_tensor * vout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_ntok, n_embd, n_layer); - vout3d->data = out; - out += ggml_nbytes(vout3d); - - ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k, - n_embd, kv_ntok, n_layer, - elt_size*n_embd, elt_size*n_embd*n_ctx, 0); - - ggml_tensor * v3d = ggml_view_3d(cpy_ctx, kv_self.v, - kv_ntok, n_embd, n_layer, - elt_size*n_ctx, elt_size*n_ctx*n_embd, 0); - - ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, k3d, kout3d)); - ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, v3d, vout3d)); - ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1); - - ggml_free(cpy_ctx); - } - } - - const size_t written = out - dst; - const size_t max_size = llama_get_state_size(ctx); - - GGML_ASSERT(written <= max_size); - - return written; -} - -// Sets the state reading from the specified source address -size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { - uint8_t * inp = src; - - // set rng - { - size_t rng_size; - char rng_buf[LLAMA_MAX_RNG_STATE]; - - memcpy(&rng_size, inp, sizeof(rng_size)); inp += sizeof(rng_size); - memcpy(&rng_buf[0], inp, LLAMA_MAX_RNG_STATE); inp += LLAMA_MAX_RNG_STATE; - - std::stringstream rng_ss; - rng_ss.str(std::string(&rng_buf[0], rng_size)); - rng_ss >> ctx->rng; - - GGML_ASSERT(rng_ss.fail() == false); - } - - // set logits - { - size_t logits_cap; - size_t logits_size; - - memcpy(&logits_cap, inp, sizeof(logits_cap)); inp += sizeof(logits_cap); - memcpy(&logits_size, inp, sizeof(logits_size)); inp += sizeof(logits_size); - - GGML_ASSERT(ctx->logits.capacity() == logits_cap); - - if (logits_size) { - ctx->logits.resize(logits_size); - memcpy(ctx->logits.data(), inp, logits_size * sizeof(float)); - } - - inp += logits_cap * sizeof(float); - } - - // set embeddings - { - size_t embedding_size; - - memcpy(&embedding_size, inp, sizeof(embedding_size)); inp += sizeof(embedding_size); - - GGML_ASSERT(ctx->embedding.capacity() == embedding_size); - - if (embedding_size) { - memcpy(ctx->embedding.data(), inp, embedding_size * sizeof(float)); - inp += embedding_size * sizeof(float); - } - } - - // set kv cache - { - const auto & kv_self = ctx->kv_self; - const auto & hparams = ctx->model.hparams; - const int n_layer = hparams.n_layer; - const int n_embd = hparams.n_embd; - const int n_ctx = hparams.n_ctx; - - size_t kv_size; - int kv_ntok; - - memcpy(&kv_size, inp, sizeof(kv_size)); inp += sizeof(kv_size); - memcpy(&kv_ntok, inp, sizeof(kv_ntok)); inp += sizeof(kv_ntok); - - if (kv_size) { - GGML_ASSERT(kv_self.buf.size == kv_size); - - const size_t elt_size = ggml_element_size(kv_self.k); - - ggml_context * cpy_ctx = ggml_init({ 4096, NULL, /* no_alloc */ true }); - ggml_cgraph gf{}; - - ggml_tensor * kin3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_ntok, n_layer); - kin3d->data = (void *) inp; - inp += ggml_nbytes(kin3d); - - ggml_tensor * vin3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_ntok, n_embd, n_layer); - vin3d->data = (void *) inp; - inp += ggml_nbytes(vin3d); - - ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k, - n_embd, kv_ntok, n_layer, - elt_size*n_embd, elt_size*n_embd*n_ctx, 0); - - ggml_tensor * v3d = ggml_view_3d(cpy_ctx, kv_self.v, - kv_ntok, n_embd, n_layer, - elt_size*n_ctx, elt_size*n_ctx*n_embd, 0); - - ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, kin3d, k3d)); - ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, vin3d, v3d)); - ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1); - - ggml_free(cpy_ctx); - } - - ctx->kv_self.n = kv_ntok; - } - - const size_t nread = inp - src; - const size_t max_size = llama_get_state_size(ctx); - - GGML_ASSERT(nread <= max_size); - - return nread; -} - -static bool llama_load_session_file_internal(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { - gguf_file file(path_session, "rb"); - GGML_UNUSED(ctx); - GGML_UNUSED(path_session); - GGML_UNUSED(tokens_out); - GGML_UNUSED(n_token_capacity); - GGML_UNUSED(n_token_count_out); - - -// TODO: implement with GGUF format - return true; -} - -bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { - try { - return llama_load_session_file_internal(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out); - } catch (const std::exception & err) { - fprintf(stderr, "error loading session file: %s\n", err.what()); - return false; - } -} - -bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { - gguf_file file(path_session, "wb"); - GGML_UNUSED(ctx); - GGML_UNUSED(tokens); - GGML_UNUSED(n_token_count); - - // TODO: implement with GGUF format - - return true; -} - -int llama_eval( - struct llama_context * ctx, - const llama_token * tokens, - int n_tokens, - int n_past, - int n_threads) { - if (!llama_eval_internal(*ctx, tokens, nullptr, n_tokens, n_past, n_threads, nullptr)) { - fprintf(stderr, "%s: failed to eval\n", __func__); - return 1; - } - - // get a more accurate load time, upon first eval - // TODO: fix this - if (!ctx->has_evaluated_once) { - ctx->t_load_us = ggml_time_us() - ctx->t_start_us; - ctx->has_evaluated_once = true; - } - - return 0; -} - - -int llama_eval_embd( - struct llama_context * ctx, - const float * embd, - int n_tokens, - int n_past, - int n_threads) { - if (!llama_eval_internal(*ctx, nullptr, embd, n_tokens, n_past, n_threads, nullptr)) { - fprintf(stderr, "%s: failed to eval\n", __func__); - return 1; - } - - // get a more accurate load time, upon first eval - // TODO: fix this - if (!ctx->has_evaluated_once) { - ctx->t_load_us = ggml_time_us() - ctx->t_start_us; - ctx->has_evaluated_once = true; - } - - return 0; -} - -int llama_eval_export(struct llama_context * ctx, const char * fname) { - const int n_batch = 1; - const int n_ctx = 512 - n_batch; - - const std::vector tmp(n_batch, llama_token_bos()); - - if (!llama_eval_internal(*ctx, tmp.data(), nullptr, tmp.size(), n_ctx, 1, fname)) { - fprintf(stderr, "%s: failed to eval\n", __func__); - return 1; - } - - return 0; -} - -int llama_tokenize_with_model( - const struct llama_model * model, - const char * text, - llama_token * tokens, - int n_max_tokens, - bool add_bos) { - auto res = llama_tokenize(model->vocab, text, add_bos); - - if (n_max_tokens < (int) res.size()) { - fprintf(stderr, "%s: too many tokens\n", __func__); - return -((int) res.size()); - } - - for (size_t i = 0; i < res.size(); i++) { - tokens[i] = res[i]; - } - - return res.size(); -} - -int llama_tokenize( - struct llama_context * ctx, - const char * text, - llama_token * tokens, - int n_max_tokens, - bool add_bos) { - return llama_tokenize_with_model(&ctx->model, text, tokens, n_max_tokens, add_bos); -} - -int llama_n_vocab_from_model(const struct llama_model * model) { - return model->vocab.id_to_token.size(); -} - -int llama_n_ctx_from_model(const struct llama_model * model) { - return model->hparams.n_ctx; -} - -int llama_n_embd_from_model(const struct llama_model * model) { - return model->hparams.n_embd; -} - -int llama_n_vocab(const struct llama_context * ctx) { - return ctx->model.vocab.id_to_token.size(); -} - -int llama_n_ctx(const struct llama_context * ctx) { - return ctx->model.hparams.n_ctx; -} - -int llama_n_embd(const struct llama_context * ctx) { - return ctx->model.hparams.n_embd; -} - -int llama_get_vocab_from_model( - const struct llama_model * model, - const char * * strings, - float * scores, - int capacity) { - int n = std::min(capacity, (int) model->vocab.id_to_token.size()); - for (int i = 0; ivocab.id_to_token[i].tok.c_str(); - scores[i] = model->vocab.id_to_token[i].score; - } - return n; -} - -int llama_get_vocab( - const struct llama_context * ctx, - const char * * strings, - float * scores, - int capacity) { - return llama_get_vocab_from_model(&ctx->model, strings, scores, capacity); -} - -float * llama_get_logits(struct llama_context * ctx) { - return ctx->logits.data(); -} - -float * llama_get_embeddings(struct llama_context * ctx) { - return ctx->embedding.data(); -} - -const char * llama_token_to_str_with_model(const struct llama_model * model, llama_token token) { - if (token >= llama_n_vocab_from_model(model)) { - return nullptr; - } - - return model->vocab.id_to_token[token].tok.c_str(); -} - -const char * llama_token_to_str(const struct llama_context * ctx, llama_token token) { - return llama_token_to_str_with_model(&ctx->model, token); -} - -llama_token llama_token_bos() { - return 1; -} - -llama_token llama_token_eos() { - return 2; -} - -llama_token llama_token_nl() { - return 13; -} - -struct llama_timings llama_get_timings(struct llama_context * ctx) { - struct llama_timings result = { - /*.t_start_ms =*/ 1e-3 * ctx->t_start_us, - /*.t_end_ms =*/ 1.00 * ggml_time_ms(), - /*.t_load_ms =*/ 1e-3 * ctx->t_load_us, - /*.t_sample_ms =*/ 1e-3 * ctx->t_sample_us, - /*.t_p_eval_ms =*/ 1e-3 * ctx->t_p_eval_us, - /*.t_eval_ms =*/ 1e-3 * ctx->t_eval_us, - - /*.n_sample =*/ std::max(1, ctx->n_sample), - /*.n_p_eval =*/ std::max(1, ctx->n_p_eval), - /*.n_eval =*/ std::max(1, ctx->n_eval), - }; - - return result; -} - -void llama_print_timings(struct llama_context * ctx) { - const llama_timings timings = llama_get_timings(ctx); - - fprintf(stderr, "\n"); - fprintf(stderr, "%s: load time = %8.2f ms\n", __func__, timings.t_load_ms); - fprintf(stderr, "%s: sample time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", - __func__, timings.t_sample_ms, timings.n_sample, timings.t_sample_ms / timings.n_sample, 1e3 / timings.t_sample_ms * timings.n_sample); - fprintf(stderr, "%s: prompt eval time = %8.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", - __func__, timings.t_p_eval_ms, timings.n_p_eval, timings.t_p_eval_ms / timings.n_p_eval, 1e3 / timings.t_p_eval_ms * timings.n_p_eval); - fprintf(stderr, "%s: eval time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", - __func__, timings.t_eval_ms, timings.n_eval, timings.t_eval_ms / timings.n_eval, 1e3 / timings.t_eval_ms * timings.n_eval); - fprintf(stderr, "%s: total time = %8.2f ms\n", __func__, (timings.t_end_ms - timings.t_start_ms)); -} - -void llama_reset_timings(struct llama_context * ctx) { - ctx->t_start_us = ggml_time_us(); - ctx->t_sample_us = ctx->n_sample = 0; - ctx->t_eval_us = ctx->n_eval = 0; - ctx->t_p_eval_us = ctx->n_p_eval = 0; -} - -const char * llama_print_system_info(void) { - static std::string s; - - s = ""; - s += "AVX = " + std::to_string(ggml_cpu_has_avx()) + " | "; - s += "AVX2 = " + std::to_string(ggml_cpu_has_avx2()) + " | "; - s += "AVX512 = " + std::to_string(ggml_cpu_has_avx512()) + " | "; - s += "AVX512_VBMI = " + std::to_string(ggml_cpu_has_avx512_vbmi()) + " | "; - s += "AVX512_VNNI = " + std::to_string(ggml_cpu_has_avx512_vnni()) + " | "; - s += "FMA = " + std::to_string(ggml_cpu_has_fma()) + " | "; - s += "NEON = " + std::to_string(ggml_cpu_has_neon()) + " | "; - s += "ARM_FMA = " + std::to_string(ggml_cpu_has_arm_fma()) + " | "; - s += "F16C = " + std::to_string(ggml_cpu_has_f16c()) + " | "; - s += "FP16_VA = " + std::to_string(ggml_cpu_has_fp16_va()) + " | "; - s += "WASM_SIMD = " + std::to_string(ggml_cpu_has_wasm_simd()) + " | "; - s += "BLAS = " + std::to_string(ggml_cpu_has_blas()) + " | "; - s += "SSE3 = " + std::to_string(ggml_cpu_has_sse3()) + " | "; - s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | "; - - return s.c_str(); -} - -// For internal test use -const std::vector>& llama_internal_get_tensor_map(struct llama_context * ctx) { - return ctx->model.tensors_by_name; -} +// Defines fileno on msys: +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#include +#include +#include +#endif + +#include "gguf-util.h" +#include "gguf-llama.h" + +#include "ggml.h" +#ifdef GGML_USE_CUBLAS +#include "ggml-cuda.h" +#elif defined(GGML_USE_CLBLAST) +#include "ggml-opencl.h" +#endif + +#ifdef GGML_USE_METAL +#include "ggml-metal.h" +#endif +#ifdef GGML_USE_MPI +#include "ggml-mpi.h" +#endif +#ifdef GGML_USE_K_QUANTS +#ifndef QK_K +#ifdef GGML_QKK_64 +#define QK_K 64 +#else +#define QK_K 256 +#endif +#endif +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +#define LLAMA_USE_SCRATCH +#define LLAMA_MAX_SCRATCH_BUFFERS 16 + +// available llama models +enum e_model { + MODEL_UNKNOWN, + MODEL_3B, + MODEL_7B, + MODEL_13B, + MODEL_30B, + MODEL_65B, + MODEL_70B, +}; + +static const size_t kB = 1024; +static const size_t MB = 1024*1024; + +// computed for n_ctx == 2048 +// TODO: dynamically determine these sizes +// needs modifications in ggml + +typedef void (*offload_func_t)(struct ggml_tensor * tensor); + +void llama_nop(struct ggml_tensor * tensor) { // don't offload by default + (void) tensor; +} + +// +// ggml helpers +// + +static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) { + struct ggml_cplan plan = ggml_graph_plan(graph, n_threads); + + if (plan.work_size > 0) { + buf.resize(plan.work_size); + plan.work_data = buf.data(); + } + + ggml_graph_compute(graph, &plan); +} + +// +// memory sizes (calculated for n_batch == 512) +// + +static const std::map & MEM_REQ_SCRATCH0(int n_ctx) +{ + static std::map k_sizes = { + { MODEL_3B, ((size_t) n_ctx / 16ull + 92ull) * MB }, + { MODEL_7B, ((size_t) n_ctx / 16ull + 100ull) * MB }, + { MODEL_13B, ((size_t) n_ctx / 12ull + 120ull) * MB }, + { MODEL_30B, ((size_t) n_ctx / 9ull + 160ull) * MB }, + { MODEL_65B, ((size_t) n_ctx / 6ull + 256ull) * MB }, // guess + { MODEL_70B, ((size_t) n_ctx / 7ull + 164ull) * MB }, + }; + return k_sizes; +} + +static const std::map & MEM_REQ_SCRATCH1() +{ + static std::map k_sizes = { + { MODEL_3B, 128ull * MB }, + { MODEL_7B, 160ull * MB }, + { MODEL_13B, 192ull * MB }, + { MODEL_30B, 256ull * MB }, + { MODEL_65B, 384ull * MB }, // guess + { MODEL_70B, 304ull * MB }, + }; + return k_sizes; +} + +// used to store the compute graph tensors + non-scratch data +static const std::map & MEM_REQ_EVAL() +{ + static std::map k_sizes = { + { MODEL_3B, 8ull * MB }, + { MODEL_7B, 10ull * MB }, + { MODEL_13B, 12ull * MB }, + { MODEL_30B, 16ull * MB }, + { MODEL_65B, 24ull * MB }, // guess + { MODEL_70B, 24ull * MB }, + }; + return k_sizes; +} + +// amount of VRAM needed per batch size to hold temporary results +// the values for 3b and 65b are not derived from testing but instead chosen conservatively +static const std::map & VRAM_REQ_SCRATCH_BASE() +{ + static std::map k_sizes = { + { MODEL_3B, 512ull * kB }, + { MODEL_7B, 512ull * kB }, + { MODEL_13B, 640ull * kB }, + { MODEL_30B, 768ull * kB }, + { MODEL_65B, 1536ull * kB }, + { MODEL_70B, 1536ull * kB }, // TODO (likely can be reduced) + }; + return k_sizes; +} + +// amount of VRAM needed per batch size and context to hold temporary results +// the values for 3b and 65b are not derived from testing but instead chosen conservatively +static const std::map & VRAM_REQ_SCRATCH_PER_CONTEXT() +{ + static std::map k_sizes = { + { MODEL_3B, 128ull }, + { MODEL_7B, 128ull }, + { MODEL_13B, 160ull }, + { MODEL_30B, 208ull }, + { MODEL_65B, 416ull }, + { MODEL_70B, 416ull }, // TODO (likely can be reduced) + }; + return k_sizes; +} + +// default hparams (LLaMA 7B) +struct llama_hparams { + uint32_t n_vocab = 32000; + uint32_t n_ctx = 512; // this is provided as user input? + uint32_t n_embd = 4096; + uint32_t n_head = 32; + uint32_t n_head_kv = 32; + uint32_t n_layer = 32; + uint32_t n_rot = 64; + uint32_t n_ff = 11008; + + float f_rms_norm_eps = LLAMA_DEFAULT_RMS_EPS; + + float rope_freq_base = 10000.0f; + float rope_freq_scale = 1.0f; + + enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16; + + bool operator!=(const llama_hparams & other) const { + return static_cast(memcmp(this, &other, sizeof(llama_hparams))); // NOLINT + } + + uint32_t n_gqa() const { + return n_head/n_head_kv; + } + + uint32_t n_embd_head() const { + return n_embd/n_head; + } + + uint32_t n_embd_gqa() const { + return n_embd/n_gqa(); + } + + size_t kv_size() const { + size_t result = 2ull; + result *= (size_t) n_embd_gqa(); + result *= (size_t) n_ctx; + result *= (size_t) n_layer; + result *= sizeof(ggml_fp16_t); + return result; + } +}; + +struct llama_layer { + // normalization + struct ggml_tensor * attention_norm; + + // attention + struct ggml_tensor * wq; + struct ggml_tensor * wk; + struct ggml_tensor * wv; + struct ggml_tensor * wo; + + // normalization + struct ggml_tensor * ffn_norm; + + // ff + struct ggml_tensor * w1; + struct ggml_tensor * w2; + struct ggml_tensor * w3; +}; + +struct llama_kv_cache { + struct ggml_tensor * k = NULL; + struct ggml_tensor * v = NULL; + + struct ggml_context * ctx = NULL; + + gguf_ctx_buffer buf; + + int n; // number of tokens currently in the cache + + ~llama_kv_cache() { + if (ctx) { + ggml_free(ctx); + } + +#ifdef GGML_USE_CUBLAS + ggml_cuda_free_data(k); + ggml_cuda_free_data(v); +#endif // GGML_USE_CUBLAS + } +}; + +struct llama_vocab { + // TODO: convert to this gguf_vocab + // add a vector of merges + // add members for bos/eos/pad/sep tokens + // so that we can pass it to different types of tokenizers with a common interface + + using id = int32_t; + using token = std::string; + + struct token_score { + token tok; + float score; + }; + + std::unordered_map token_to_id; + std::vector id_to_token; +}; + +struct llama_model { + e_model type = MODEL_UNKNOWN; + + llama_hparams hparams; + + struct ggml_tensor * tok_embeddings; + + struct ggml_tensor * norm; + struct ggml_tensor * output; + + std::vector layers; + int n_gpu_layers; + + // context + struct ggml_context * ctx = NULL; + + // the model memory buffer + gguf_ctx_buffer buf; + + // model memory mapped file + std::unique_ptr mapping; + + // objects representing data potentially being locked in memory + gguf_mlock mlock_buf; + gguf_mlock mlock_mmap; + + // for quantize-stats only + std::vector> tensors_by_name; + + int64_t t_load_us = 0; + int64_t t_start_us = 0; + + llama_vocab vocab; + + ~llama_model() { + if (ctx) { + ggml_free(ctx); + } + +#ifdef GGML_USE_CUBLAS + for (size_t i = 0; i < tensors_by_name.size(); ++i) { + ggml_cuda_free_data(tensors_by_name[i].second); + } + ggml_cuda_free_scratch(); +#elif defined(GGML_USE_CLBLAST) + for (size_t i = 0; i < tensors_by_name.size(); ++i) { + ggml_cl_free_data(tensors_by_name[i].second); + } +#endif + } +}; + +struct llama_context { + llama_context(const llama_model & model) : model(model), t_load_us(model.t_load_us), t_start_us(model.t_start_us) {} +#ifdef GGML_USE_METAL + ~llama_context() { + if (ctx_metal) { + ggml_metal_free(ctx_metal); + } + } +#endif + std::mt19937 rng; + + bool has_evaluated_once = false; + + int64_t t_sample_us = 0; + int64_t t_eval_us = 0; + int64_t t_p_eval_us = 0; + + int32_t n_sample = 0; // number of tokens sampled + int32_t n_eval = 0; // number of eval calls + int32_t n_p_eval = 0; // number of tokens in eval calls for the prompt (with batch size > 1) + + const llama_model & model; + + bool model_owner = false; + + int64_t t_load_us; + int64_t t_start_us; + + // key + value cache for the self attention + struct llama_kv_cache kv_self; + + size_t mem_per_token = 0; + + // decode output (2-dimensional array: [n_tokens][n_vocab]) + std::vector logits; + bool logits_all = false; + + // input embedding (1-dimensional array: [n_embd]) + std::vector embedding; + + // reusable buffer for `struct ggml_graph_plan.work_data` + std::vector work_buffer; + + // memory buffers used to evaluate the model + // TODO: move in llama_state + gguf_ctx_buffer buf_compute; + gguf_ctx_buffer buf_scratch[LLAMA_MAX_SCRATCH_BUFFERS]; + +#ifdef GGML_USE_METAL + ggml_metal_context * ctx_metal = NULL; +#endif + +#ifdef GGML_USE_MPI + ggml_mpi_context * ctx_mpi = NULL; +#endif + + int buf_last = 0; + size_t buf_max_size[LLAMA_MAX_SCRATCH_BUFFERS] = { 0 }; + + void use_buf(struct ggml_context * ctx, int i) { +#if defined(LLAMA_USE_SCRATCH) + size_t last_size = 0; + + if (i == -1) { + last_size = ggml_set_scratch(ctx, { 0, 0, nullptr, }); + } else { + auto & buf = buf_scratch[i]; + last_size = ggml_set_scratch(ctx, { 0, buf.size, buf.addr, }); + } + + if (buf_last >= 0) { + buf_max_size[buf_last] = std::max(buf_max_size[buf_last], last_size); + } + + buf_last = i; +#else + (void) i; + (void) ctx; +#endif + } + + size_t get_buf_max_mem(int i) const { +#if defined(LLAMA_USE_SCRATCH) + return buf_max_size[i]; +#else + (void) i; + return 0; +#endif + } +}; + +template +static T checked_mul(T a, T b) { + T ret = a * b; + if (a != 0 && ret / a != b) { + throw std::runtime_error(format("overflow multiplying %llu * %llu", + (unsigned long long) a, (unsigned long long) b)); + } + return ret; +} + +static size_t checked_div(size_t a, size_t b) { + if (b == 0 || a % b != 0) { + throw std::runtime_error(format("error dividing %zu / %zu", a, b)); + } + return a / b; +} + +static std::string llama_format_tensor_shape(const std::vector & ne) { + char buf[256]; + snprintf(buf, sizeof(buf), "%5u", ne.at(0)); + for (size_t i = 1; i < ne.size(); i++) { + snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), " x %5u", ne.at(i)); + } + return buf; +} + +static size_t llama_calc_tensor_size(const std::vector & ne, enum ggml_type type) { + size_t size = ggml_type_size(type); + for (uint32_t dim : ne) { + size = checked_mul(size, dim); + } + return size / ggml_blck_size(type); +} + +struct gguf_load_tensor { + std::string name; + enum ggml_type type = GGML_TYPE_F32; + std::vector ne; + size_t file_off; + size_t size; + struct ggml_tensor * ggml_tensor = NULL; + uint8_t * data; +}; + +struct gguf_load_tensors_map { + // tensors is kept in a separate vector to preserve file order + std::vector tensors; + std::unordered_map name_to_idx; +}; + +enum gguf_file_version { + GGUF_FILE_VERSION_V1 = 1, + +}; + + +struct gguf_file_loader { + gguf_file file; + gguf_context * gguf_ctx; + gguf_file_version file_version; + llama_hparams hparams; + llama_vocab vocab; +struct ggml_context * ctx_data = NULL; + + gguf_file_loader(const char * fname, gguf_load_tensors_map & tensors_map) + : file(fname, "rb") { + fprintf(stderr, "llama.cpp: loading model from %s\n", fname); + + struct gguf_init_params params = { + /*.no_alloc = */ true, + /*.ctx = */ &ctx_data, + }; + + gguf_ctx = gguf_init_from_file(fname, params); + file_version = (enum gguf_file_version) gguf_get_version(gguf_ctx); + + read_hparams(); + read_vocab(); + read_tensor_metadata(tensors_map); + } + + uint32_t read_u32(const char * key) { + int i = gguf_find_key(gguf_ctx, key); + if (i == -1) { + throw std::runtime_error(format("cannot find param with key %s\n", key)); + } + + return gguf_get_val_u32(gguf_ctx, i); + } + + float read_f32(const char * key) { + int i = gguf_find_key(gguf_ctx, key); + if (i == -1) { + throw std::runtime_error(format("cannot find param with key %s\n", key)); + } + + return gguf_get_val_f32(gguf_ctx, i); + } + + int read_n_vocab() { + int i = gguf_find_key(gguf_ctx, "tokenizer.ggml.tokens"); + if (i == -1) { + throw std::runtime_error("cannot find token list in GGUF file\n"); + } + + return gguf_get_arr_n(gguf_ctx, i); + } + + void read_hparams() { + + // TODO define keys as constants in header + // TODO: read all hparams from file + + hparams.n_vocab = read_n_vocab(); + hparams.n_ctx = read_u32("llama.context_length"); + hparams.n_embd = read_u32("llama.embedding_length"); + hparams.n_ff = read_u32("llama.feed_forward_length"); + hparams.n_head = read_u32("llama.attention.head_count"); + hparams.n_layer = read_u32("llama.layer_count"); + hparams.n_rot = read_u32("llama.rope.dimension_count"); + hparams.f_rms_norm_eps = read_f32("llama.attention.layer_norm_rms_epsilon"); + + // LLaMAv2 + // hparams.n_head_kv = read_u32("llama.attention.head_count_kv"); + } + + void read_vocab() { + vocab.id_to_token.resize(hparams.n_vocab); + int token_idx = gguf_find_key(gguf_ctx, "tokenizer.ggml.tokens"); + if (token_idx == -1) { + throw std::runtime_error("cannot find token list in GGUF file\n"); + } + + int score_idx = gguf_find_key(gguf_ctx, "tokenizer.ggml.scores"); + if (score_idx == -1) { + throw std::runtime_error("cannot find token scores list in GGUF file\n"); + } + + for (uint32_t i = 0; i < hparams.n_vocab; i++) { + + std::string word = gguf_get_arr_str(gguf_ctx, token_idx, i); + + vocab.token_to_id[word] = i; + + auto & tok_score = vocab.id_to_token[i]; + tok_score.tok = std::move(word); + tok_score.score = gguf_get_arr_f32(gguf_ctx, score_idx, i); + } + } + + void read_tensor_metadata(gguf_load_tensors_map & tensors_map) { + const int n_tensors = gguf_get_n_tensors(gguf_ctx); + + for (int i = 0; i < n_tensors; ++i) { + gguf_load_tensor tensor; + const char * name = gguf_get_tensor_name(gguf_ctx, i); + + struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); + uint32_t n_dims = cur->n_dims; + tensor.type = cur->type; + tensor.ne.resize(n_dims); + for (uint32_t j = 0; j < n_dims; ++j) { + tensor.ne[j] = cur->ne[j]; + } + + if (n_dims < 1 || n_dims > 2) { + throw std::runtime_error(format("llama.cpp: tensor '%s' should not be %u-dimensional", name, n_dims)); + } + switch (tensor.type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + break; + default: { + throw std::runtime_error(format("unrecognized tensor type %u\n", tensor.type)); + } + } + + + tensor.file_off = gguf_get_data_offset(gguf_ctx) + gguf_get_tensor_offset(gguf_ctx, i); + + tensor.name = name; + tensor.size = llama_calc_tensor_size(tensor.ne, tensor.type); + + tensors_map.tensors.push_back(tensor); + tensors_map.name_to_idx[name] = tensors_map.tensors.size() - 1; + } + } +}; + +struct gguf_file_saver { + // TODO + // this implementation now assumes that the data section is of the same length as the unquantized model. + // this is needed to write tensor metadata and weights in a single pass by seeking to appropriate positions in the file. + // this may not be true when we add quantization version and change ftype description (currently it's string according to the specs, + // but better to have it as uint32). + // we need to calculate the delta in number of bytes written with a counter as a struct member. + + gguf_file file; + gguf_file_loader * fl; + size_t info_offset; + size_t tensor_offset = 0; + + gguf_file_saver(const char * fname, gguf_file_loader * fl, enum llama_ftype new_ftype) + : file(fname, "wb"), fl(fl) { + fprintf(stderr, "llama.cpp: saving model to %s\n", fname); + write_header(); + write_hparams(new_ftype); + } + + void write_header() { + const int32_t magic = GGUF_MAGIC; + file.write_i32(magic); + + const int32_t version = GGUF_VERSION; + file.write_i32(version); + + const int32_t n_tensors = gguf_get_n_tensors(fl->gguf_ctx); + file.write_i32(n_tensors); + + const int32_t n_kv = gguf_get_n_kv(fl->gguf_ctx); + file.write_i32(n_kv); + } + + void write_hparam_arr_str(const std::string & key, enum gguf_type type, int i, int n_arr) { + std::vector data(n_arr); + + for (int j = 0; j < n_arr; ++j) { + std::string val = gguf_get_arr_str(fl->gguf_ctx, i, j); + data[j] = val; + } + + file.write_arr(key, type, data); + } + + void write_hparam_arr_f32(const std::string & key, enum gguf_type type, int i, int n_arr) { + std::vector data(n_arr); + + for (int j = 0; j < n_arr; ++j) { + float val = gguf_get_arr_f32(fl->gguf_ctx, i, j); + data[j] = val; + } + + file.write_arr(key, type, data); + } + + void write_hparams(enum llama_ftype new_ftype) { + const int32_t n_kv = gguf_get_n_kv(fl->gguf_ctx); + for (int i = 0; i < n_kv; ++i) { + const char * key = gguf_get_key(fl->gguf_ctx, i); + if (strcmp(key, "general.quantization_version") == 0) { + file.write_val("general.quantization_version", GGUF_TYPE_UINT32, new_ftype); + } else { + const gguf_type vtype = gguf_get_kv_type(fl->gguf_ctx, i); + + bool bool_val; + float f32_val; + int16_t i16_val; + int32_t i32_val; + int8_t i8_val; + std::string str_val; + uint16_t u16_val; + uint32_t u32_val; + uint8_t u8_val; + gguf_type arr_type; + int n_arr; + + switch(vtype) { + case GGUF_TYPE_BOOL: + bool_val = gguf_get_val_bool(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_BOOL, bool_val); + break; + case GGUF_TYPE_FLOAT32: + f32_val = gguf_get_val_f32(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_FLOAT32, f32_val); + break; + case GGUF_TYPE_INT16: + i16_val = gguf_get_val_i16(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_INT16, i16_val); + break; + case GGUF_TYPE_INT32: + i32_val = gguf_get_val_i32(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_INT32, i32_val); + break; + case GGUF_TYPE_INT8: + i8_val = gguf_get_val_i8(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_INT8, i8_val); + break; + case GGUF_TYPE_STRING: + str_val = gguf_get_val_str(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_STRING, str_val); + break; + case GGUF_TYPE_UINT16: + u16_val = gguf_get_val_u16(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_UINT16, u16_val); + break; + case GGUF_TYPE_UINT32: + u32_val = gguf_get_val_u32(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_UINT32, u32_val); + break; + case GGUF_TYPE_UINT8: + u8_val = gguf_get_val_u8(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_UINT8, u8_val); + break; + case GGUF_TYPE_ARRAY: + arr_type = gguf_get_arr_type(fl->gguf_ctx, i); + n_arr = gguf_get_arr_n(fl->gguf_ctx, i); + if (arr_type == GGUF_TYPE_FLOAT32) { + write_hparam_arr_f32(key, arr_type, i, n_arr); + } else if (arr_type == GGUF_TYPE_STRING) { + write_hparam_arr_str(key, GGUF_TYPE_STRING, i, n_arr); + } else { + throw std::runtime_error("not implemented"); + } + break; + default: + throw std::runtime_error(format("cannot recognize value type for key %s\n", key)); + } + } + } + + info_offset = file.tell(); + size_t count = gguf_get_data_offset(fl->gguf_ctx) - info_offset; + file.write_zeros(count); + file.seek(info_offset, SEEK_SET); + GGML_ASSERT(info_offset == file.tell()); + } + + size_t write_tensor_info(gguf_load_tensor & tensor, enum ggml_type type) { + size_t total_written = 0; + file.seek(info_offset, SEEK_SET); + GGML_ASSERT(info_offset == file.tell()); + total_written += file.write_str(tensor.name); + + int32_t n_dims = tensor.ne.size(); + total_written += file.write_i32(n_dims); + for (int32_t i = 0; i < n_dims; ++i) { + total_written += file.write_i32(tensor.ne[i]); + } + + total_written += file.write_i32(type); + total_written += file.write_u64(tensor_offset); + info_offset += total_written; // position to write info of the next tensor + + file.seek(0, SEEK_END); + + return total_written; + } + + void write_tensor(gguf_load_tensor & tensor, enum ggml_type new_type, const void * new_data, size_t new_size) { + switch (new_type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + break; + default: GGML_ASSERT(false); + } + + write_tensor_info(tensor, new_type); + file.write_raw(new_data, new_size); + size_t padded_size = GGML_PAD(new_size, GGUF_DEFAULT_ALIGNMENT); // TODO: handle custom alignment + size_t pad = padded_size - new_size; + file.write_zeros(pad); + tensor_offset += padded_size; // offset of the next tensor + } +}; + +struct llama_model_loader { + std::unique_ptr file_loader; + gguf_load_tensors_map tensors_map; + bool use_mmap; + size_t num_ggml_tensors_created = 0; + struct ggml_context * ggml_ctx = NULL; + std::unique_ptr mapping; + + llama_model_loader(const std::string & fname_base, bool use_mmap) { + file_loader = std::unique_ptr(new gguf_file_loader(fname_base.c_str(), tensors_map)); + if (!gguf_mmap::SUPPORTED) { + use_mmap = false; + } + this->use_mmap = use_mmap; + } + + void calc_sizes(size_t * ctx_size_p, size_t * mmapped_size_p) const { + *ctx_size_p = *mmapped_size_p = 0; + for (const gguf_load_tensor & lt : tensors_map.tensors) { + *ctx_size_p += sizeof(struct ggml_tensor) + GGML_OBJECT_SIZE; + *(use_mmap ? mmapped_size_p : ctx_size_p) += lt.size + 16; + } + } + + struct ggml_tensor * get_tensor(const std::string & name, const std::vector & ne, ggml_backend backend) { + auto it = tensors_map.name_to_idx.find(name); + if (it == tensors_map.name_to_idx.end()) { + throw std::runtime_error(std::runtime_error(format("llama.cpp: tensor '%s' is missing from model", name.c_str()))); + } + gguf_load_tensor & lt = tensors_map.tensors.at(it->second); + if (lt.ne != ne) { + throw std::runtime_error(format("llama.cpp: tensor '%s' has wrong shape; expected %s, got %s", + name.c_str(), llama_format_tensor_shape(ne).c_str(), llama_format_tensor_shape(lt.ne).c_str())); + } + + return get_tensor_for(lt, backend); + } + + struct ggml_tensor * get_tensor_for(gguf_load_tensor & lt, ggml_backend backend) { + struct ggml_tensor * tensor; + if (backend != GGML_BACKEND_CPU) { + ggml_set_no_alloc(ggml_ctx, true); + } + if (lt.ne.size() == 2) { + tensor = ggml_new_tensor_2d(ggml_ctx, lt.type, lt.ne.at(0), lt.ne.at(1)); + } else { + GGML_ASSERT(lt.ne.size() == 1); + tensor = ggml_new_tensor_1d(ggml_ctx, lt.type, lt.ne.at(0)); + } + ggml_set_name(tensor, lt.name.c_str()); + GGML_ASSERT(lt.ggml_tensor == NULL); // if this fails, we called get_tensor twice on the same tensor + + if (backend != GGML_BACKEND_CPU) { + ggml_set_no_alloc(ggml_ctx, use_mmap); + } + tensor->backend = backend; + lt.ggml_tensor = tensor; + num_ggml_tensors_created++; + return tensor; + } + + void done_getting_tensors() const { + if (num_ggml_tensors_created != tensors_map.tensors.size()) { + throw std::runtime_error(std::string("llama.cpp: file contained more tensors than expected")); + } + } + + void load_all_data(llama_progress_callback progress_callback, void * progress_callback_user_data, gguf_mlock * lmlock) { + size_t data_size = 0; + size_t prefetch_size = 0; + size_t lock_size = 0; + for (const gguf_load_tensor & lt : tensors_map.tensors) { + data_size += lt.size; + if (lt.ggml_tensor->backend == GGML_BACKEND_CPU) { + prefetch_size += lt.size; + } + } + + if (use_mmap) { + mapping.reset(new gguf_mmap(&file_loader->file, prefetch_size, ggml_is_numa())); + if (lmlock) { + lmlock->init(mapping->addr); + } + } + + size_t done_size = 0; + for (gguf_load_tensor & lt : tensors_map.tensors) { + if (progress_callback) { + progress_callback((float) done_size / data_size, progress_callback_user_data); + } + GGML_ASSERT(lt.ggml_tensor); // unused tensors should have been caught by load_data already + lt.data = (uint8_t *) lt.ggml_tensor->data; + + // allocate temp buffer if not using mmap + if (!use_mmap && lt.data == NULL) { + GGML_ASSERT(lt.ggml_tensor->backend != GGML_BACKEND_CPU); + lt.data = (uint8_t*)malloc(ggml_nbytes(lt.ggml_tensor)); + } + + load_data_for(lt); + + switch(lt.ggml_tensor->backend) { + case GGML_BACKEND_CPU: + lt.ggml_tensor->data = lt.data; + if (use_mmap && lmlock) { + lock_size += lt.size; + lmlock->grow_to(lock_size); + } + break; +#if defined(GGML_USE_CUBLAS) + case GGML_BACKEND_GPU: + case GGML_BACKEND_GPU_SPLIT: + ggml_cuda_transform_tensor(lt.data, lt.ggml_tensor); + if (!use_mmap) { + free(lt.data); + } + break; +#elif defined(GGML_USE_CLBLAST) + case GGML_BACKEND_GPU: + ggml_cl_transform_tensor(lt.data, lt.ggml_tensor); + if (!use_mmap) { + free(lt.data); + } + break; +#endif + default: + continue; + } + + done_size += lt.size; + } + } + + void load_data_for(gguf_load_tensor & lt) { + if (use_mmap) { + lt.data = (uint8_t *) mapping->addr + lt.file_off; + } else { + gguf_file & file = file_loader->file; + file.seek(lt.file_off, SEEK_SET); + file.read_raw(lt.data, lt.size); + } + + if (0) { + print_checksum(lt); + } + } + + static void print_checksum(gguf_load_tensor & lt) { + uint32_t sum = 0; + for (size_t i = 0; i < lt.size; i++) { + uint8_t byte = lt.data[i]; + sum = byte + (sum << 6) + (sum << 16) - sum; // sdbm hash + } + fprintf(stderr, "%s checksum: %#08x (%s, size %zu)\n", lt.name.c_str(), sum, + llama_format_tensor_shape(lt.ne).c_str(), lt.size); + } + +}; + +// +// kv cache +// + +static bool kv_cache_init( + const struct llama_hparams & hparams, + struct llama_kv_cache & cache, + ggml_type wtype, + int n_ctx, + int n_gpu_layers) { + const int n_embd = hparams.n_embd_gqa(); + const int n_layer = hparams.n_layer; + + const int64_t n_mem = n_layer*n_ctx; + const int64_t n_elements = n_embd*n_mem; + + cache.buf.resize(2u*n_elements*ggml_type_size(wtype) + 2u*MB); + cache.n = 0; + + struct ggml_init_params params; + params.mem_size = cache.buf.size; + params.mem_buffer = cache.buf.addr; + params.no_alloc = false; + + cache.ctx = ggml_init(params); + + if (!cache.ctx) { + fprintf(stderr, "%s: failed to allocate memory for kv cache\n", __func__); + return false; + } + + cache.k = ggml_new_tensor_1d(cache.ctx, wtype, n_elements); + cache.v = ggml_new_tensor_1d(cache.ctx, wtype, n_elements); + ggml_set_name(cache.k, "cache_k"); + ggml_set_name(cache.v, "cache_v"); + + (void) n_gpu_layers; +#ifdef GGML_USE_CUBLAS + if (n_gpu_layers > n_layer + 1) { + ggml_cuda_assign_buffers_no_scratch(cache.v); + } + if (n_gpu_layers > n_layer + 2) { + ggml_cuda_assign_buffers_no_scratch(cache.k); + } +#endif // GGML_USE_CUBLAS + + return true; +} + +struct llama_context_params llama_context_default_params() { + struct llama_context_params result = { + /*.seed =*/ LLAMA_DEFAULT_SEED, + /*.n_ctx =*/ 512, + /*.n_batch =*/ 512, + /*.n_gqa =*/ 1, + /*.rms_norm_eps =*/ LLAMA_DEFAULT_RMS_EPS, + /*.gpu_layers =*/ 0, + /*.main_gpu =*/ 0, + /*.tensor_split =*/ nullptr, + /*.rope_freq_base =*/ 10000.0f, + /*.rope_freq_scale =*/ 1.0f, + /*.progress_callback =*/ nullptr, + /*.progress_callback_user_data =*/ nullptr, + /*.low_vram =*/ false, + /*.f16_kv =*/ true, + /*.logits_all =*/ false, + /*.vocab_only =*/ false, + /*.use_mmap =*/ true, + /*.use_mlock =*/ false, + /*.embedding =*/ false, + }; + + return result; +} + +struct llama_model_quantize_params llama_model_quantize_default_params() { + struct llama_model_quantize_params result = { + /*.nthread =*/ 0, + /*.ftype =*/ LLAMA_FTYPE_MOSTLY_Q5_1, + /*.allow_requantize =*/ false, + /*.quantize_output_tensor =*/ true, + }; + + return result; +} + +int llama_max_devices() { + return LLAMA_MAX_DEVICES; +} + +bool llama_mmap_supported() { + return gguf_mmap::SUPPORTED; +} + +bool llama_mlock_supported() { + return gguf_mlock::SUPPORTED; +} + +void llama_backend_init(bool numa) { + ggml_time_init(); + + // needed to initialize f16 tables + { + struct ggml_init_params params = { 0, NULL, false }; + struct ggml_context * ctx = ggml_init(params); + ggml_free(ctx); + } + + if (numa) { + ggml_numa_init(); + } + +#ifdef GGML_USE_MPI + ggml_mpi_backend_init(); +#endif +} + +void llama_backend_free() { +#ifdef GGML_USE_MPI + ggml_mpi_backend_free(); +#endif +} + +int64_t llama_time_us() { + return ggml_time_us(); +} + +// +// model loading +// + +static const char *gguf_file_version_name(gguf_file_version version) { + switch (version) { + case GGUF_FILE_VERSION_V1: return "GGUF V1 (latest)"; + } + + return "unknown"; +} + +static const char *llama_ftype_name(enum llama_ftype ftype) { + switch (ftype) { + case LLAMA_FTYPE_ALL_F32: return "all F32"; + case LLAMA_FTYPE_MOSTLY_F16: return "mostly F16"; + case LLAMA_FTYPE_MOSTLY_Q4_0: return "mostly Q4_0"; + case LLAMA_FTYPE_MOSTLY_Q4_1: return "mostly Q4_1"; + case LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16: + return "mostly Q4_1, some F16"; + case LLAMA_FTYPE_MOSTLY_Q5_0: return "mostly Q5_0"; + case LLAMA_FTYPE_MOSTLY_Q5_1: return "mostly Q5_1"; + case LLAMA_FTYPE_MOSTLY_Q8_0: return "mostly Q8_0"; + // K-quants + case LLAMA_FTYPE_MOSTLY_Q2_K: return "mostly Q2_K"; + case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "mostly Q3_K - Small"; + case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "mostly Q3_K - Medium"; + case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "mostly Q3_K - Large"; + case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "mostly Q4_K - Small"; + case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "mostly Q4_K - Medium"; + case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "mostly Q5_K - Small"; + case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "mostly Q5_K - Medium"; + case LLAMA_FTYPE_MOSTLY_Q6_K: return "mostly Q6_K"; + default: return "unknown, may not work"; + } +} + +static const char *llama_model_type_name(e_model type) { + switch (type) { + case MODEL_3B: return "3B"; + case MODEL_7B: return "7B"; + case MODEL_13B: return "13B"; + case MODEL_30B: return "30B"; + case MODEL_65B: return "65B"; + case MODEL_70B: return "70B"; + default: GGML_ASSERT(false); + } +} + +static void llama_model_load_internal( + const std::string & fname, + llama_model & model, + llama_vocab & vocab, + int n_ctx, + int n_batch, + int n_gqa, + float rms_norm_eps, + int n_gpu_layers, + int main_gpu, + const float * tensor_split, + float rope_freq_base, + float rope_freq_scale, + bool low_vram, + ggml_type memory_type, + bool use_mmap, + bool use_mlock, + bool vocab_only, + llama_progress_callback progress_callback, + void * progress_callback_user_data) { + GGML_UNUSED(rms_norm_eps); // TODO: update function signature to remove this + + model.t_start_us = ggml_time_us(); + + std::unique_ptr ml(new llama_model_loader(fname, use_mmap)); + + vocab = std::move(ml->file_loader->vocab); + model.hparams = ml->file_loader->hparams; + model.n_gpu_layers = n_gpu_layers; + gguf_file_version file_version = ml->file_loader->file_version; + + auto & hparams = model.hparams; + + { + switch (hparams.n_layer) { + case 26: model.type = e_model::MODEL_3B; break; + case 32: model.type = e_model::MODEL_7B; break; + case 40: model.type = e_model::MODEL_13B; break; + case 60: model.type = e_model::MODEL_30B; break; + case 80: model.type = e_model::MODEL_65B; break; + default: + { + if (hparams.n_layer < 32) { + model.type = e_model::MODEL_7B; + } + } break; + } + + hparams.n_ctx = n_ctx; + + // LLaMAv2 + hparams.n_head_kv = hparams.n_head / n_gqa; + if (model.type == e_model::MODEL_65B && n_gqa == 8) { + fprintf(stderr, "%s: warning: assuming 70B model based on GQA == %d\n", __func__, n_gqa); + model.type = e_model::MODEL_70B; + } + + hparams.rope_freq_base = rope_freq_base; + hparams.rope_freq_scale = rope_freq_scale; + } + + const uint32_t n_ff = hparams.n_ff; + + { + fprintf(stderr, "%s: format = %s\n", __func__, gguf_file_version_name(file_version)); + fprintf(stderr, "%s: n_vocab = %u\n", __func__, hparams.n_vocab); + fprintf(stderr, "%s: n_ctx = %u\n", __func__, hparams.n_ctx); + fprintf(stderr, "%s: n_embd = %u\n", __func__, hparams.n_embd); + fprintf(stderr, "%s: n_head = %u\n", __func__, hparams.n_head); + fprintf(stderr, "%s: n_head_kv = %u\n", __func__, hparams.n_head_kv); + fprintf(stderr, "%s: n_layer = %u\n", __func__, hparams.n_layer); + fprintf(stderr, "%s: n_rot = %u\n", __func__, hparams.n_rot); // a.k.a. n_embd_head, n_head_dim + fprintf(stderr, "%s: n_gqa = %u\n", __func__, hparams.n_gqa()); + fprintf(stderr, "%s: rnorm_eps = %.1e\n", __func__, hparams.f_rms_norm_eps); + fprintf(stderr, "%s: n_ff = %u\n", __func__, n_ff); + fprintf(stderr, "%s: freq_base = %.1f\n", __func__, hparams.rope_freq_base); + fprintf(stderr, "%s: freq_scale = %g\n", __func__, hparams.rope_freq_scale); + fprintf(stderr, "%s: ftype = %u (%s)\n", __func__, hparams.ftype, llama_ftype_name(hparams.ftype)); + fprintf(stderr, "%s: model size = %s\n", __func__, llama_model_type_name(model.type)); + } + + if (hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || + hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_1 || + hparams.ftype == LLAMA_FTYPE_MOSTLY_Q8_0) { + throw std::runtime_error(format("this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1508)")); + } + + if (vocab_only) { + return; + } + + auto & ctx = model.ctx; + + size_t ctx_size; + size_t mmapped_size; + ml->calc_sizes(&ctx_size, &mmapped_size); + fprintf(stderr, "%s: ggml ctx size = %7.2f MB\n", __func__, ctx_size/1024.0/1024.0); + + // create the ggml context + { + model.buf.resize(ctx_size); + if (use_mlock) { + model.mlock_buf.init (model.buf.addr); + model.mlock_buf.grow_to(model.buf.size); + } + + struct ggml_init_params params = { + /*.mem_size =*/ model.buf.size, + /*.mem_buffer =*/ model.buf.addr, + /*.no_alloc =*/ ml->use_mmap, + }; + + model.ctx = ggml_init(params); + if (!model.ctx) { + throw std::runtime_error(format("ggml_init() failed")); + } + } + + (void) main_gpu; +#if defined(GGML_USE_CUBLAS) + fprintf(stderr, "%s: using CUDA for GPU acceleration\n", __func__); + ggml_cuda_set_main_device(main_gpu); +#define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU +#define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_GPU_SPLIT +#elif defined(GGML_USE_CLBLAST) + fprintf(stderr, "%s: using OpenCL for GPU acceleration\n", __func__); +#define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU +#define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_GPU +#else +#define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_CPU +#define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_CPU +#endif + + // prepare memory for the weights + size_t vram_weights = 0; + size_t vram_scratch = 0; + { + const uint32_t n_embd = hparams.n_embd; + const uint32_t n_embd_gqa = hparams.n_embd_gqa(); + const uint32_t n_layer = hparams.n_layer; + const uint32_t n_vocab = hparams.n_vocab; + + ml->ggml_ctx = ctx; + + model.tok_embeddings = ml->get_tensor("tok_embeddings.weight", {n_embd, n_vocab}, GGML_BACKEND_CPU); + + // "output" tensor + { + ggml_backend backend_norm; + ggml_backend backend_output; + if (n_gpu_layers > int(n_layer)) { // NOLINT + // norm is not performance relevant on its own but keeping it in VRAM reduces data copying + // on Windows however this is detrimental unless everything is on the GPU +#ifndef _WIN32 + backend_norm = low_vram ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD; +#else + backend_norm = low_vram || n_gpu_layers <= (int) n_layer + 2 ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD; +#endif // _WIN32 + + backend_output = LLAMA_BACKEND_OFFLOAD_SPLIT; + } else { + backend_norm = GGML_BACKEND_CPU; + backend_output = GGML_BACKEND_CPU; + } + + model.norm = ml->get_tensor("norm.weight", {n_embd}, backend_norm); + model.output = ml->get_tensor("output.weight", {n_embd, n_vocab}, backend_output); + if (backend_norm == GGML_BACKEND_GPU) { + vram_weights += ggml_nbytes(model.norm); + } + if (backend_output == GGML_BACKEND_GPU_SPLIT) { + vram_weights += ggml_nbytes(model.output); + } + } + + const int i_gpu_start = n_layer - n_gpu_layers; + + model.layers.resize(n_layer); + for (uint32_t i = 0; i < n_layer; ++i) { + const ggml_backend backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD; // NOLINT + const ggml_backend backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : LLAMA_BACKEND_OFFLOAD_SPLIT; // NOLINT + + auto & layer = model.layers[i]; + + std::string layers_i = "layers." + std::to_string(i); + + layer.attention_norm = ml->get_tensor(layers_i + ".attention_norm.weight", {n_embd}, backend); + + layer.wq = ml->get_tensor(layers_i + ".attention.wq.weight", {n_embd, n_embd}, backend_split); + layer.wk = ml->get_tensor(layers_i + ".attention.wk.weight", {n_embd, n_embd_gqa}, backend_split); + layer.wv = ml->get_tensor(layers_i + ".attention.wv.weight", {n_embd, n_embd_gqa}, backend_split); + layer.wo = ml->get_tensor(layers_i + ".attention.wo.weight", {n_embd, n_embd}, backend_split); + + layer.ffn_norm = ml->get_tensor(layers_i + ".ffn_norm.weight", {n_embd}, backend); + + layer.w1 = ml->get_tensor(layers_i + ".feed_forward.w1.weight", {n_embd, n_ff}, backend_split); + layer.w2 = ml->get_tensor(layers_i + ".feed_forward.w2.weight", { n_ff, n_embd}, backend_split); + layer.w3 = ml->get_tensor(layers_i + ".feed_forward.w3.weight", {n_embd, n_ff}, backend_split); + + if (backend == GGML_BACKEND_GPU) { + vram_weights += + ggml_nbytes(layer.attention_norm) + ggml_nbytes(layer.wq) + ggml_nbytes(layer.wk) + + ggml_nbytes(layer.wv) + ggml_nbytes(layer.wo) + ggml_nbytes(layer.ffn_norm) + + ggml_nbytes(layer.w1) + ggml_nbytes(layer.w2) + ggml_nbytes(layer.w3); + } + } + } + + ml->done_getting_tensors(); + + // print memory requirements + { + const size_t scale = memory_type == GGML_TYPE_F32 ? 2 : 1; + + // this is the total memory required to run the inference + const size_t mem_required = + ctx_size + + mmapped_size - vram_weights + // weights in VRAM not in memory + MEM_REQ_SCRATCH0(hparams.n_ctx).at(model.type) + + MEM_REQ_SCRATCH1().at(model.type) + + MEM_REQ_EVAL().at(model.type); + + // this is the memory required by one llama_state + const size_t mem_required_state = + scale*hparams.kv_size(); + + fprintf(stderr, "%s: mem required = %7.2f MB (+ %7.2f MB per state)\n", __func__, + mem_required / 1024.0 / 1024.0, mem_required_state / 1024.0 / 1024.0); + + (void) vram_scratch; + (void) n_batch; +#ifdef GGML_USE_CUBLAS + if (low_vram) { + fprintf(stderr, "%s: not allocating a VRAM scratch buffer due to low VRAM option\n", __func__); + ggml_cuda_set_scratch_size(0); // disable scratch + } else { + const size_t vram_scratch_base = VRAM_REQ_SCRATCH_BASE().at(model.type); + const size_t vram_scratch_per_context = VRAM_REQ_SCRATCH_PER_CONTEXT().at(model.type); + vram_scratch = n_batch * (vram_scratch_base + n_ctx * vram_scratch_per_context); + ggml_cuda_set_scratch_size(vram_scratch); + if (n_gpu_layers > 0) { + fprintf(stderr, "%s: allocating batch_size x (%zd kB + n_ctx x %zd B) = %zd MB VRAM for the scratch buffer\n", + __func__, vram_scratch_base / kB, vram_scratch_per_context, + (vram_scratch + MB - 1) / MB); // round up + } + } +#endif // GGML_USE_CUBLAS + +#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) + const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer)); + + fprintf(stderr, "%s: offloading %d repeating layers to GPU\n", __func__, n_gpu); + if (n_gpu_layers > (int) hparams.n_layer) { + fprintf(stderr, "%s: offloading non-repeating layers to GPU\n", __func__); + } + size_t vram_kv_cache = 0; + +#ifdef GGML_USE_CUBLAS + const int max_backend_supported_layers = hparams.n_layer + 3; + const int max_offloadable_layers = low_vram ? hparams.n_layer + 1 : hparams.n_layer + 3; + if (n_gpu_layers > (int) hparams.n_layer + 1) { + if (low_vram) { + fprintf(stderr, "%s: cannot offload v cache to GPU due to low VRAM option\n", __func__); + } else { + fprintf(stderr, "%s: offloading v cache to GPU\n", __func__); + vram_kv_cache += hparams.kv_size() / 2; + } + } + if (n_gpu_layers > (int) hparams.n_layer + 2) { + if (low_vram) { + fprintf(stderr, "%s: cannot offload k cache to GPU due to low VRAM option\n", __func__); + } else { + fprintf(stderr, "%s: offloading k cache to GPU\n", __func__); + vram_kv_cache += hparams.kv_size() / 2; + } + } +#elif defined(GGML_USE_CLBLAST) + const int max_backend_supported_layers = hparams.n_layer + 1; + const int max_offloadable_layers = hparams.n_layer + 1; +#endif // GGML_USE_CUBLAS + + fprintf(stderr, "%s: offloaded %d/%d layers to GPU\n", + __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers); + fprintf(stderr, "%s: total VRAM used: %zu MB\n", + __func__, (vram_weights + vram_scratch + vram_kv_cache + MB - 1) / MB); // round up +#else + (void) n_gpu_layers; +#endif // defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) + } + + // populate `tensors_by_name` + for (gguf_load_tensor & lt : ml->tensors_map.tensors) { + model.tensors_by_name.emplace_back(lt.name, lt.ggml_tensor); + } + + (void) tensor_split; +#if defined(GGML_USE_CUBLAS) + { + ggml_cuda_set_tensor_split(tensor_split); + } +#endif + + ml->load_all_data(progress_callback, progress_callback_user_data, use_mlock ? &model.mlock_mmap : NULL); + + if (progress_callback) { + progress_callback(1.0f, progress_callback_user_data); + } + + model.mapping = std::move(ml->mapping); + + // loading time will be recalculate after the first eval, so + // we take page faults deferred by mmap() into consideration + model.t_load_us = ggml_time_us() - model.t_start_us; +} + +static bool llama_model_load( + const std::string & fname, + llama_model & model, + llama_vocab & vocab, + int n_ctx, + int n_batch, + int n_gqa, + float rms_norm_eps, + int n_gpu_layers, + int main_gpu, + const float * tensor_split, + float rope_freq_base, + float rope_freq_scale, + bool low_vram, + ggml_type memory_type, + bool use_mmap, + bool use_mlock, + bool vocab_only, + llama_progress_callback progress_callback, + void *progress_callback_user_data) { + try { + llama_model_load_internal(fname, model, vocab, n_ctx, n_batch, n_gqa, rms_norm_eps, n_gpu_layers, main_gpu, tensor_split, rope_freq_base, rope_freq_scale, low_vram, memory_type, + use_mmap, use_mlock, vocab_only, progress_callback, progress_callback_user_data); + return true; + } catch (const std::exception & err) { + fprintf(stderr, "error loading model: %s\n", err.what()); + return false; + } +} + +// evaluate the transformer +// +// - lctx: llama context +// - tokens: new batch of tokens to process +// - embd embeddings input +// - n_tokens number of tokens +// - n_past: the context size so far +// - n_threads: number of threads to use +// +static bool llama_eval_internal( + llama_context & lctx, + const llama_token * tokens, + const float * embd, + int n_tokens, + int n_past, + int n_threads, + const char * cgraph_fname) { + + GGML_ASSERT((!tokens && embd) || (tokens && !embd)); + +#ifdef GGML_USE_MPI + ggml_mpi_eval_init(lctx.ctx_mpi, &n_tokens, &n_past, &n_threads); +#endif + + const int64_t t_start_us = ggml_time_us(); + + const int N = n_tokens; + + const auto & model = lctx.model; + const auto & hparams = model.hparams; + + const auto & kv_self = lctx.kv_self; + + GGML_ASSERT(!!kv_self.ctx); + + const int64_t n_embd = hparams.n_embd; + const int64_t n_layer = hparams.n_layer; + const int64_t n_ctx = hparams.n_ctx; + const int64_t n_head = hparams.n_head; + const int64_t n_head_kv = hparams.n_head_kv; + const int64_t n_embd_head = hparams.n_embd_head(); + const int64_t n_vocab = hparams.n_vocab; + const int64_t n_embd_gqa = hparams.n_embd_gqa(); + + + GGML_ASSERT(n_embd_head == hparams.n_rot); + + const float freq_base = hparams.rope_freq_base; + const float freq_scale = hparams.rope_freq_scale; + const float rms_norm_eps = hparams.f_rms_norm_eps; + + const int n_gpu_layers = model.n_gpu_layers; + + auto & mem_per_token = lctx.mem_per_token; + auto & buf_compute = lctx.buf_compute; + + struct ggml_init_params params = { + /*.mem_size =*/ buf_compute.size, + /*.mem_buffer =*/ buf_compute.addr, + /*.no_alloc =*/ false, + }; + + struct ggml_context * ctx0 = ggml_init(params); + + ggml_cgraph * gf = ggml_new_graph(ctx0); + + // for big prompts, if BLAS is enabled, it is better to use only one thread + // otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance + n_threads = N >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas() ? 1 : n_threads; + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + if (tokens) { + struct ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); + memcpy(inp_tokens->data, tokens, N*ggml_element_size(inp_tokens)); + ggml_set_name(inp_tokens, "inp_tokens"); + + inpL = ggml_get_rows(ctx0, model.tok_embeddings, inp_tokens); + } else { +#ifdef GGML_USE_MPI + GGML_ASSERT(false && "not implemented"); +#endif + + inpL = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N); + memcpy(inpL->data, embd, N * n_embd * ggml_element_size(inpL)); + } + + const int i_gpu_start = n_layer - n_gpu_layers; + (void) i_gpu_start; + + // offload functions set the tensor output backend to GPU + // tensors are GPU-accelerated if any input or the output has been offloaded + // + // with the low VRAM option VRAM scratch is disabled in llama_load_model_internal + // in that case ggml_cuda_assign_buffers has no effect + offload_func_t offload_func_nr = llama_nop; // nr = non-repeating + offload_func_t offload_func_kq = llama_nop; + offload_func_t offload_func_v = llama_nop; + +#ifdef GGML_USE_CUBLAS + if (n_gpu_layers > n_layer) { + offload_func_nr = ggml_cuda_assign_buffers; + } + if (n_gpu_layers > n_layer + 1) { + offload_func_v = ggml_cuda_assign_buffers; + } + if (n_gpu_layers > n_layer + 2) { + offload_func_kq = ggml_cuda_assign_buffers; + } +#endif // GGML_USE_CUBLAS + + for (int il = 0; il < n_layer; ++il) { + ggml_format_name(inpL, "layer_inp_%d", il); + + offload_func_t offload_func = llama_nop; + +#ifdef GGML_USE_CUBLAS + if (il >= i_gpu_start) { + offload_func = ggml_cuda_assign_buffers; + } +#endif // GGML_USE_CUBLAS + + struct ggml_tensor * inpSA = inpL; + + lctx.use_buf(ctx0, 0); + + // norm + { + cur = ggml_rms_norm(ctx0, inpL, rms_norm_eps); + offload_func(cur); + ggml_set_name(cur, "rms_norm_0"); + + // cur = cur*attention_norm(broadcasted) + cur = ggml_mul(ctx0, cur, model.layers[il].attention_norm); + offload_func(cur); + ggml_set_name(cur, "attention_norm_0"); + } + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * tmpk = ggml_mul_mat(ctx0, model.layers[il].wk, cur); + offload_func_kq(tmpk); + ggml_set_name(tmpk, "tmpk"); + + struct ggml_tensor * tmpq = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + offload_func_kq(tmpq); + ggml_set_name(tmpq, "tmpq"); + + struct ggml_tensor * Kcur = ggml_rope_custom_inplace(ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, N), n_past, n_embd_head, 0, 0, freq_base, freq_scale); + offload_func_kq(Kcur); + ggml_set_name(Kcur, "Kcur"); + + struct ggml_tensor * Qcur = ggml_rope_custom_inplace(ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, N), n_past, n_embd_head, 0, 0, freq_base, freq_scale); + offload_func_kq(Qcur); + ggml_set_name(Qcur, "Qcur"); + + // store key and value to memory + { + // compute the transposed [N, n_embd] V matrix + + struct ggml_tensor * tmpv = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + offload_func_v(tmpv); + ggml_set_name(tmpv, "tmpv"); + + struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_reshape_2d(ctx0, tmpv, n_embd_gqa, N)); + offload_func_v(Vcur); + ggml_set_name(Vcur, "Vcur"); + + struct ggml_tensor * k = ggml_view_1d(ctx0, kv_self.k, N*n_embd_gqa, (ggml_element_size(kv_self.k)*n_embd_gqa)*(il*n_ctx + n_past)); + offload_func_kq(k); + ggml_set_name(k, "k"); + + struct ggml_tensor * v = ggml_view_2d(ctx0, kv_self.v, N, n_embd_gqa, + ( n_ctx)*ggml_element_size(kv_self.v), + (il*n_ctx)*ggml_element_size(kv_self.v)*n_embd_gqa + n_past*ggml_element_size(kv_self.v)); + offload_func_v(v); + ggml_set_name(v, "v"); + + // important: storing RoPE-ed version of K in the KV cache! + ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k)); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v)); + } + + struct ggml_tensor * Q = + ggml_permute(ctx0, + Qcur, + 0, 2, 1, 3); + offload_func_kq(Q); + ggml_set_name(Q, "Q"); + + struct ggml_tensor * K = + ggml_permute(ctx0, + ggml_reshape_3d(ctx0, + ggml_view_1d(ctx0, kv_self.k, (n_past + N)*n_embd_gqa, il*n_ctx*ggml_element_size(kv_self.k)*n_embd_gqa), + n_embd_head, n_head_kv, n_past + N), + 0, 2, 1, 3); + offload_func_kq(K); + ggml_set_name(K, "K"); + + // K * Q + struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); + offload_func_kq(KQ); + ggml_set_name(KQ, "KQ"); + + // KQ_scaled = KQ / sqrt(n_embd_head) + struct ggml_tensor * KQ_scale = ggml_new_f32(ctx0, 1.0f/sqrtf(float(n_embd)/n_head)); + ggml_set_name(KQ_scale, "1/sqrt(n_embd_head)"); + + // KQ_scaled shape [n_past + N, N, n_head, 1] + struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, KQ_scale); + offload_func_kq(KQ_scaled); + ggml_set_name(KQ_scaled, "KQ_scaled"); + + // KQ_masked = mask_past(KQ_scaled) + struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past); + offload_func_kq(KQ_masked); + ggml_set_name(KQ_masked, "KQ_masked"); + + // KQ = soft_max(KQ_masked) + struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked); + offload_func_v(KQ_soft_max); + ggml_set_name(KQ_soft_max, "KQ_soft_max"); + + // split cached V into n_head heads + struct ggml_tensor * V = + ggml_view_3d(ctx0, kv_self.v, + n_past + N, n_embd_head, n_head_kv, + n_ctx*ggml_element_size(kv_self.v), + n_ctx*ggml_element_size(kv_self.v)*n_embd_head, + n_ctx*ggml_element_size(kv_self.v)*n_embd_gqa*il); + offload_func_v(V); + ggml_set_name(V, "V"); + +#if 1 + struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max); + offload_func_v(KQV); + ggml_set_name(KQV, "KQV"); +#else + // make V contiguous in memory to speed up the matmul, however we waste time on the copy + // on M1 this is faster for the perplexity computation, but ~5% slower for the single-token generation + // is there a better way? + struct ggml_tensor * V_cont = ggml_cpy(ctx0, V, ggml_new_tensor_3d(ctx0, kv_self.v->type, n_past + N, n_embd_head, n_head)); + struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_cont, KQ_soft_max); +#endif + + // KQV_merged = KQV.permute(0, 2, 1, 3) + struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3); + offload_func_v(KQV_merged); + ggml_set_name(KQV_merged, "KQV_merged"); + + // cur = KQV_merged.contiguous().view(n_embd, N) + cur = ggml_cpy(ctx0, + KQV_merged, + ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N)); + offload_func_v(cur); + ggml_set_name(cur, "KQV_merged_contiguous"); + + // projection (no bias) + cur = ggml_mul_mat(ctx0, + model.layers[il].wo, + cur); + offload_func(cur); + ggml_set_name(cur, "result_wo"); + } + + lctx.use_buf(ctx0, 1); + + struct ggml_tensor * inpFF = ggml_add(ctx0, cur, inpSA); + offload_func(inpFF); + ggml_set_name(inpFF, "inpFF"); + + // feed-forward network + { + // norm + { + cur = ggml_rms_norm(ctx0, inpFF, rms_norm_eps); + offload_func(cur); + ggml_set_name(cur, "rms_norm_1"); + + // cur = cur*ffn_norm(broadcasted) + cur = ggml_mul(ctx0, cur, model.layers[il].ffn_norm); + offload_func(cur); + ggml_set_name(cur, "ffn_norm"); + } + + struct ggml_tensor * tmp = ggml_mul_mat(ctx0, + model.layers[il].w3, + cur); + offload_func(tmp); + ggml_set_name(tmp, "result_w3"); + + cur = ggml_mul_mat(ctx0, + model.layers[il].w1, + cur); + offload_func(cur); + ggml_set_name(cur, "result_w1"); + + // SILU activation + cur = ggml_silu(ctx0, cur); + offload_func(cur); + ggml_set_name(cur, "silu"); + + cur = ggml_mul(ctx0, cur, tmp); + offload_func(cur); + ggml_set_name(cur, "silu_x_result_w3"); + + cur = ggml_mul_mat(ctx0, + model.layers[il].w2, + cur); + offload_func(cur); + ggml_set_name(cur, "result_w2"); + } + + cur = ggml_add(ctx0, cur, inpFF); + offload_func(cur); + ggml_set_name(cur, "inpFF_+_result_w2"); + + // input for next layer + inpL = cur; + } + + lctx.use_buf(ctx0, 0); + + // used at the end to optionally extract the embeddings + struct ggml_tensor * embeddings = NULL; + + // norm + { + cur = ggml_rms_norm(ctx0, inpL, rms_norm_eps); + offload_func_nr(cur); + ggml_set_name(cur, "rms_norm_2"); + + // cur = cur*norm(broadcasted) + cur = ggml_mul(ctx0, cur, model.norm); + // offload_func_nr(cur); // TODO CPU + GPU mirrored backend + ggml_set_name(cur, "result_norm"); + + embeddings = cur; + } + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + ggml_set_name(cur, "result_output"); + + lctx.use_buf(ctx0, -1); + + // logits -> probs + //cur = ggml_soft_max_inplace(ctx0, cur); + + // run the computation + ggml_build_forward_expand(gf, cur); + + // fprintf(stderr, "graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf.n_nodes, gf.n_leafs); + +#if GGML_USE_MPI + ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer); +#endif + +#ifdef GGML_USE_METAL + if (lctx.ctx_metal && N == 1) { + if (!ggml_metal_if_optimized(lctx.ctx_metal)) { + ggml_metal_graph_find_concurrency(lctx.ctx_metal, gf); + } + ggml_metal_set_n_cb (lctx.ctx_metal, n_threads); + ggml_metal_graph_compute(lctx.ctx_metal, gf); + ggml_metal_get_tensor (lctx.ctx_metal, cur); + } else { + // IMPORTANT: + // Since we don't have efficient Matrix x Matrix Metal multiplication yet, we fallback to vanilla + // ggml_graph_compute(). It uses Apple's Accelerate CBLAS API which takes advantage of the ANE or the AMX + // coprocessor. + // + // When we implement Matrix x Matrix Metal multiplication, we can avoid this branch. + // But for now, we have focused only on Matrix x Vector Metal multiplication. + // + // TODO: avoid these syncs via shared memory (ref #1696) + // + if (lctx.ctx_metal) { + // We need to sync the GPU KV cache with the CPU KV cache + ggml_metal_get_tensor(lctx.ctx_metal, kv_self.k); + ggml_metal_get_tensor(lctx.ctx_metal, kv_self.v); + } + + ggml_graph_compute_helper(lctx.work_buffer, gf, n_threads); + } +#else + ggml_graph_compute_helper(lctx.work_buffer, gf, n_threads); +#endif + +#if GGML_USE_MPI + ggml_mpi_graph_compute_post(lctx.ctx_mpi, gf, n_layer); +#endif + + // update kv token count + lctx.kv_self.n = n_past + N; + + struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; + + if (cgraph_fname) { + ggml_graph_export(gf, cgraph_fname); + } + +#ifdef GGML_PERF + // print timing information per ggml operation (for debugging purposes) + // requires GGML_PERF to be defined + ggml_graph_print(gf); +#endif + + // plot the computation graph in dot format (for debugging purposes) + //if (n_past%100 == 0) { + // ggml_graph_dump_dot(gf, NULL, "llama.dot"); + //} + + // extract logits + { + auto & logits_out = lctx.logits; + + if (lctx.logits_all) { + logits_out.resize(n_vocab * N); + memcpy(logits_out.data(), (float *) ggml_get_data(res), sizeof(float)*n_vocab*N); + } else { + // return result for just the last token + logits_out.resize(n_vocab); + memcpy(logits_out.data(), (float *) ggml_get_data(res) + (n_vocab*(N-1)), sizeof(float)*n_vocab); + } + } + + // extract embeddings + if (!lctx.embedding.empty()) { + auto & embedding_out = lctx.embedding; + + embedding_out.resize(n_embd); + memcpy(embedding_out.data(), (float *) ggml_get_data(embeddings) + (n_embd*(N - 1)), sizeof(float)*n_embd); + } + + if (mem_per_token == 0) { + mem_per_token = ggml_used_mem(ctx0)/N; + } + +#if 0 + printf("\n%s: used_mem: eval ctx %.3f MB, scratch %.3f MB %.3f MB, work buf %.3f MB, n_past = %d, N = %d\n", __func__, + ggml_used_mem(ctx0)/1024.0/1024.0, + lctx.get_buf_max_mem(0)/1024.0/1024.0, + lctx.get_buf_max_mem(1)/1024.0/1024.0, + lctx.work_buffer.size()/1024.0/1024.0, + n_past, N); +#endif + + ggml_free(ctx0); + + // measure the performance only for the single-token evals + if (N == 1) { + lctx.t_eval_us += ggml_time_us() - t_start_us; + lctx.n_eval++; + } + else if (N > 1) { + lctx.t_p_eval_us += ggml_time_us() - t_start_us; + lctx.n_p_eval += N; + } + + return true; +} + +// +// tokenizer +// + +static size_t utf8_len(char src) { + const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; + uint8_t highbits = static_cast(src) >> 4; + return lookup[highbits]; +} + +struct llama_sp_symbol { + using index = int; + index prev; + index next; + const char * text; + size_t n; +}; + +static_assert(std::is_trivially_copyable::value, "llama_sp_symbol is not trivially copyable"); + +struct llama_sp_bigram { + struct comparator { + bool operator()(llama_sp_bigram & l, llama_sp_bigram & r) { + return (l.score < r.score) || (l.score == r.score && l.left > r.left); + } + }; + using queue_storage = std::vector; + using queue = std::priority_queue; + llama_sp_symbol::index left; + llama_sp_symbol::index right; + float score; + size_t size; +}; + +// original implementation: +// https://github.com/ggerganov/llama.cpp/commit/074bea2eb1f1349a0118239c4152914aecaa1be4 +struct llama_tokenizer { + llama_tokenizer(const llama_vocab & vocab): vocab_(vocab) {} + + void tokenize(const std::string & text, std::vector & output) { + // split string into utf8 chars + int index = 0; + size_t offs = 0; + while (offs < text.size()) { + llama_sp_symbol sym; + size_t char_len = std::min(text.size() - offs, utf8_len(text[offs])); + sym.text = text.c_str() + offs; + sym.n = char_len; + offs += char_len; + sym.prev = index - 1; + sym.next = offs == text.size() ? -1 : index + 1; + index++; + symbols_.emplace_back(sym); + } + + // seed the work queue with all possible 2-character tokens. + for (size_t i = 1; i < symbols_.size(); ++i) { + try_add_bigram(i - 1, i); + } + + // keep substituting the highest frequency pairs for as long as we can. + while (!work_queue_.empty()) { + auto bigram = work_queue_.top(); + work_queue_.pop(); + + auto & left_sym = symbols_[bigram.left]; + auto & right_sym = symbols_[bigram.right]; + + // if one of the symbols already got merged, skip it. + if (left_sym.n == 0 || right_sym.n == 0 || + left_sym.n + right_sym.n != bigram.size) { + continue; + } + + // merge the right sym into the left one + left_sym.n += right_sym.n; + right_sym.n = 0; + + //printf("left = '%*s' size = %zu\n", (int) left_sym.n, left_sym.text, bigram.size); + + // remove the right sym from the chain + left_sym.next = right_sym.next; + if (right_sym.next >= 0) { + symbols_[right_sym.next].prev = bigram.left; + } + + // find more substitutions + try_add_bigram(left_sym.prev, bigram.left); + try_add_bigram(bigram.left, left_sym.next); + } + + for (int i = 0; i != -1; i = symbols_[i].next) { + auto & symbol = symbols_[i]; + auto token = vocab_.token_to_id.find(std::string(symbol.text, symbol.n)); + + if (token == vocab_.token_to_id.end()) { + // output any symbols that did not form tokens as bytes. + for (int j = 0; j < (int) symbol.n; ++j) { + llama_vocab::id token_id = static_cast(symbol.text[j]) + 3; + output.push_back(token_id); + } + } else { + output.push_back((*token).second); + } + } + } + +private: + void try_add_bigram(int left, int right) { + if (left == -1 || right == -1) { + return; + } + + const std::string text = std::string(symbols_[left].text, symbols_[left].n + symbols_[right].n); + auto token = vocab_.token_to_id.find(text); + + if (token == vocab_.token_to_id.end()) { + return; + } + + if (static_cast((*token).second) >= vocab_.id_to_token.size()) { + return; + } + + const auto &tok_score = vocab_.id_to_token[(*token).second]; + + llama_sp_bigram bigram; + bigram.left = left; + bigram.right = right; + bigram.score = tok_score.score; + bigram.size = text.size(); + work_queue_.push(bigram); + } + + const llama_vocab & vocab_; + std::vector symbols_; + llama_sp_bigram::queue work_queue_; +}; + +static std::vector llama_tokenize(const llama_vocab & vocab, const std::string & text, bool bos) { + llama_tokenizer tokenizer(vocab); + std::vector output; + + if (text.empty()) { + return output; + } + + if (bos) { + output.push_back(llama_token_bos()); + } + + tokenizer.tokenize(text, output); + return output; +} + +// +// grammar - internal +// + +struct llama_grammar { + const std::vector> rules; + std::vector> stacks; +}; + +struct llama_grammar_candidate { + size_t index; + const uint32_t * code_points; +}; + +// NOTE: assumes valid utf8 (but checks for overrun) +// adds a terminating 0 for use as pointer +std::vector decode_utf8(const char * src) { + static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; + const char * pos = src; + std::vector code_points; + while (*pos != 0) { + uint8_t first_byte = static_cast(*pos); + uint8_t highbits = first_byte >> 4; + int len = lookup[highbits]; + uint8_t mask = (1 << (8 - len)) - 1; + uint32_t value = first_byte & mask; + const char * end = pos + len; // may overrun! + ++pos; + for ( ; pos < end && *pos != 0; ++pos) { + value = (value << 6) + (static_cast(*pos) & 0x3F); + } + code_points.push_back(value); + } + code_points.push_back(0); + return code_points; +} + +// returns true iff pos points to the end of one of the definitions of a rule +static bool llama_grammar_is_end_of_sequence(const llama_grammar_element * pos) { + switch (pos->type) { + case LLAMA_GRETYPE_END: return true; + case LLAMA_GRETYPE_ALT: return true; + default: return false; + } +} + +// returns true iff chr satisfies the char range at pos (regular or inverse range) +// asserts that pos is pointing to a char range element +static std::pair llama_grammar_match_char( + const llama_grammar_element * pos, + const uint32_t chr) { + + bool found = false; + bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR; + GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT); + + do { + if (pos[1].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) { + // inclusive range, e.g. [a-z] + found = found || (pos->value <= chr && chr <= pos[1].value); + pos += 2; + } else { + // exact char match, e.g. [a] or "a" + found = found || pos->value == chr; + pos += 1; + } + } while (pos->type == LLAMA_GRETYPE_CHAR_ALT); + + return std::make_pair(found == is_positive_char, pos); +} + +// transforms a grammar pushdown stack into N possible stacks, all ending +// at a character range (terminal element) +static void llama_grammar_advance_stack( + const std::vector> & rules, + const std::vector & stack, + std::vector> & new_stacks) { + + if (stack.empty()) { + new_stacks.push_back(stack); + return; + } + + const llama_grammar_element * pos = stack.back(); + + switch (pos->type) { + case LLAMA_GRETYPE_RULE_REF: { + const size_t rule_id = static_cast(pos->value); + const llama_grammar_element * subpos = rules[rule_id].data(); + do { + // init new stack without the top (pos) + std::vector new_stack(stack.begin(), stack.end() - 1); + if (!llama_grammar_is_end_of_sequence(pos + 1)) { + // if this rule ref is followed by another element, add that to stack + new_stack.push_back(pos + 1); + } + if (!llama_grammar_is_end_of_sequence(subpos)) { + // if alternate is nonempty, add to stack + new_stack.push_back(subpos); + } + llama_grammar_advance_stack(rules, new_stack, new_stacks); + while (!llama_grammar_is_end_of_sequence(subpos)) { + // scan to end of alternate def + subpos++; + } + if (subpos->type == LLAMA_GRETYPE_ALT) { + // there's another alternate def of this rule to process + subpos++; + } else { + break; + } + } while (true); + break; + } + case LLAMA_GRETYPE_CHAR: + case LLAMA_GRETYPE_CHAR_NOT: + new_stacks.push_back(stack); + break; + default: + // end of alternate (LLAMA_GRETYPE_END, LLAMA_GRETYPE_ALT) or middle of char range + // (LLAMA_GRETYPE_CHAR_ALT, LLAMA_GRETYPE_CHAR_RNG_UPPER); stack should never be left on + // those + GGML_ASSERT(false); + } +} + +// takes a set of possible pushdown stacks on a grammar, which are required to +// be positioned at a character range (see `llama_grammar_advance_stack`), and +// produces the N possible stacks if the given char is accepted at those +// positions +static std::vector> llama_grammar_accept( + const std::vector> & rules, + const std::vector> & stacks, + const uint32_t chr) { + + std::vector> new_stacks; + + for (const auto & stack : stacks) { + if (stack.empty()) { + continue; + } + + auto match = llama_grammar_match_char(stack.back(), chr); + if (match.first) { + const llama_grammar_element * pos = match.second; + + // update top of stack to next element, if any + std::vector new_stack(stack.begin(), stack.end() - 1); + if (!llama_grammar_is_end_of_sequence(pos)) { + new_stack.push_back(pos); + } + llama_grammar_advance_stack(rules, new_stack, new_stacks); + } + } + + return new_stacks; +} + +static std::vector llama_grammar_reject_candidates( + const std::vector> & rules, + const std::vector> & stacks, + const std::vector & candidates); + +static std::vector llama_grammar_reject_candidates_for_stack( + const std::vector> & rules, + const std::vector & stack, + const std::vector & candidates) { + + std::vector rejects; + + if (stack.empty()) { + // accept nothing; EOS is handled elsewhere + rejects.insert(rejects.end(), candidates.begin(), candidates.end()); + return rejects; + } + + const llama_grammar_element * stack_pos = stack.back(); + + std::vector next_candidates; + for (auto tok : candidates) { + if (llama_grammar_match_char(stack_pos, tok.code_points[0]).first) { + if (tok.code_points[1] != 0) { + next_candidates.push_back({ tok.index, tok.code_points + 1 }); + } + } else { + rejects.push_back(tok); + } + } + + auto stack_pos_after = llama_grammar_match_char(stack_pos, 0).second; + + // update top of stack to next element, if any + std::vector stack_after(stack.begin(), stack.end() - 1); + if (!llama_grammar_is_end_of_sequence(stack_pos_after)) { + stack_after.push_back(stack_pos_after); + } + std::vector> next_stacks; + llama_grammar_advance_stack(rules, stack_after, next_stacks); + + auto next_rejects = llama_grammar_reject_candidates(rules, next_stacks, next_candidates); + for (auto tok : next_rejects) { + rejects.push_back({ tok.index, tok.code_points - 1 }); + } + + return rejects; +} + +static std::vector llama_grammar_reject_candidates( + const std::vector> & rules, + const std::vector> & stacks, + const std::vector & candidates) { + GGML_ASSERT(!stacks.empty()); // REVIEW + + if (candidates.empty()) { + return std::vector(); + } + + auto rejects = llama_grammar_reject_candidates_for_stack(rules, stacks.front(), candidates); + + for (size_t i = 1, size = stacks.size(); i < size; ++i) { + rejects = llama_grammar_reject_candidates_for_stack(rules, stacks[i], rejects); + } + return rejects; +} + +// +// grammar - external +// + +struct llama_grammar * llama_grammar_init( + const llama_grammar_element ** rules, + size_t n_rules, + size_t start_rule_index) { + const llama_grammar_element * pos; + + // copy rule definitions into vectors + std::vector> vec_rules(n_rules); + for (size_t i = 0; i < n_rules; i++) { + for (pos = rules[i]; pos->type != LLAMA_GRETYPE_END; pos++) { + vec_rules[i].push_back(*pos); + } + vec_rules[i].push_back({LLAMA_GRETYPE_END, 0}); + } + + // loop over alternates of start rule to build initial stacks + std::vector> stacks; + pos = rules[start_rule_index]; + do { + std::vector stack; + if (!llama_grammar_is_end_of_sequence(pos)) { + // if alternate is nonempty, add to stack + stack.push_back(pos); + } + llama_grammar_advance_stack(vec_rules, stack, stacks); + while (!llama_grammar_is_end_of_sequence(pos)) { + // scan to end of alternate def + pos++; + } + if (pos->type == LLAMA_GRETYPE_ALT) { + // there's another alternate def of this rule to process + pos++; + } else { + break; + } + } while (true); + + return new llama_grammar{ std::move(vec_rules), std::move(stacks) }; +} + +void llama_grammar_free(struct llama_grammar * grammar) { + delete grammar; +} + +// +// sampling +// + +void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates) { + assert(candidates->size > 0); + + const int64_t t_start_sample_us = ggml_time_us(); + + // Sort the logits in descending order + if (!candidates->sorted) { + std::sort(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) { + return a.logit > b.logit; + }); + candidates->sorted = true; + } + + float max_l = candidates->data[0].logit; + float cum_sum = 0.0f; + for (size_t i = 0; i < candidates->size; ++i) { + float p = expf(candidates->data[i].logit - max_l); + candidates->data[i].p = p; + cum_sum += p; + } + for (size_t i = 0; i < candidates->size; ++i) { + candidates->data[i].p /= cum_sum; + } + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep) { + const int64_t t_start_sample_us = ggml_time_us(); + + k = std::max(k, (int) min_keep); + k = std::min(k, (int) candidates->size); + + // Sort scores in descending order + if (!candidates->sorted) { + auto comp = [](const llama_token_data & a, const llama_token_data & b) { + return a.logit > b.logit; + }; + if (k == (int) candidates->size) { + std::sort(candidates->data, candidates->data + candidates->size, comp); + } else { + std::partial_sort(candidates->data, candidates->data + k, candidates->data + candidates->size, comp); + } + candidates->sorted = true; + } + candidates->size = k; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) { + if (p >= 1.0f) { + return; + } + + llama_sample_softmax(ctx, candidates); + + const int64_t t_start_sample_us = ggml_time_us(); + + // Compute the cumulative probabilities + float cum_sum = 0.0f; + size_t last_idx = candidates->size; + + for (size_t i = 0; i < candidates->size; ++i) { + cum_sum += candidates->data[i].p; + + // Check if the running sum is at least p or if we have kept at least min_keep tokens + // we set the last index to i+1 to indicate that the current iterate should be included in the set + if (cum_sum >= p && i + 1 >= min_keep) { + last_idx = i + 1; + break; + } + } + + // Resize the output vector to keep only the top-p tokens + candidates->size = last_idx; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep) { + if (z >= 1.0f || candidates->size <= 2) { + return; + } + + llama_sample_softmax(nullptr, candidates); + const int64_t t_start_sample_us = ggml_time_us(); + + // Compute the first and second derivatives + std::vector first_derivatives(candidates->size - 1); + std::vector second_derivatives(candidates->size - 2); + + for (size_t i = 0; i < first_derivatives.size(); ++i) { + first_derivatives[i] = candidates->data[i].p - candidates->data[i + 1].p; + } + for (size_t i = 0; i < second_derivatives.size(); ++i) { + second_derivatives[i] = first_derivatives[i] - first_derivatives[i + 1]; + } + + // Calculate absolute value of second derivatives + for (size_t i = 0; i < second_derivatives.size(); ++i) { + second_derivatives[i] = abs(second_derivatives[i]); + } + + // Normalize the second derivatives + { + const float second_derivatives_sum = std::accumulate(second_derivatives.begin(), second_derivatives.end(), 0.0f); + + if (second_derivatives_sum > 1e-6f) { + for (float & value : second_derivatives) { + value /= second_derivatives_sum; + } + } else { + for (float & value : second_derivatives) { + value = 1.0f / second_derivatives.size(); + } + } + } + + float cum_sum = 0.0f; + size_t last_idx = candidates->size; + for (size_t i = 0; i < second_derivatives.size(); ++i) { + cum_sum += second_derivatives[i]; + + // Check if the running sum is greater than z or if we have kept at least min_keep tokens + if (cum_sum > z && i >= min_keep) { + last_idx = i; + break; + } + } + + // Resize the output vector to keep only the tokens above the tail location + candidates->size = last_idx; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + + +void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) { + // Reference implementation: + // https://github.com/huggingface/transformers/compare/main...cimeister:typical-sampling:typical-pr + if (p >= 1.0f) { + return; + } + + // Compute the softmax of logits and calculate entropy + llama_sample_softmax(nullptr, candidates); + + const int64_t t_start_sample_us = ggml_time_us(); + + float entropy = 0.0f; + for (size_t i = 0; i < candidates->size; ++i) { + entropy += -candidates->data[i].p * logf(candidates->data[i].p); + } + + // Compute the absolute difference between negative log probability and entropy for each candidate + std::vector shifted_scores; + for (size_t i = 0; i < candidates->size; ++i) { + float shifted_score = fabsf(-logf(candidates->data[i].p) - entropy); + shifted_scores.push_back(shifted_score); + } + + // Sort tokens based on the shifted_scores and their corresponding indices + std::vector indices(candidates->size); + std::iota(indices.begin(), indices.end(), 0); + + std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) { + return shifted_scores[a] < shifted_scores[b]; + }); + + // Compute the cumulative probabilities + float cum_sum = 0.0f; + size_t last_idx = indices.size(); + + for (size_t i = 0; i < indices.size(); ++i) { + size_t idx = indices[i]; + cum_sum += candidates->data[idx].p; + + // Check if the running sum is greater than typical or if we have kept at least min_keep tokens + if (cum_sum > p && i >= min_keep - 1) { + last_idx = i + 1; + break; + } + } + + // Resize the output vector to keep only the locally typical tokens + std::vector new_candidates; + for (size_t i = 0; i < last_idx; ++i) { + size_t idx = indices[i]; + new_candidates.push_back(candidates->data[idx]); + } + + // Replace the data in candidates with the new_candidates data + std::copy(new_candidates.begin(), new_candidates.end(), candidates->data); + candidates->size = new_candidates.size(); + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates_p, float temp) { + const int64_t t_start_sample_us = ggml_time_us(); + + for (size_t i = 0; i < candidates_p->size; ++i) { + candidates_p->data[i].logit /= temp; + } + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_repetition_penalty(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float penalty) { + if (last_tokens_size == 0 || penalty == 1.0f) { + return; + } + + const int64_t t_start_sample_us = ggml_time_us(); + + for (size_t i = 0; i < candidates->size; ++i) { + const auto * token_iter = std::find(last_tokens, last_tokens + last_tokens_size, candidates->data[i].id); + if (token_iter == last_tokens + last_tokens_size) { + continue; + } + + // The academic publication that described this technique actually just only divided, but that would cause tokens with negative logits to become more likely, which is obviously wrong. + // This is common fix for this problem, which is to multiply by the penalty instead of dividing. + if (candidates->data[i].logit <= 0) { + candidates->data[i].logit *= penalty; + } else { + candidates->data[i].logit /= penalty; + } + } + + candidates->sorted = false; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_frequency_and_presence_penalties(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens_p, size_t last_tokens_size, float alpha_frequency, float alpha_presence) { + if (last_tokens_size == 0 || (alpha_frequency == 0.0f && alpha_presence == 0.0f)) { + return; + } + + const int64_t t_start_sample_us = ggml_time_us(); + + // Create a frequency map to count occurrences of each token in last_tokens + std::unordered_map token_count; + for (size_t i = 0; i < last_tokens_size; ++i) { + token_count[last_tokens_p[i]]++; + } + + // Apply frequency and presence penalties to the candidates + for (size_t i = 0; i < candidates->size; ++i) { + auto token_iter = token_count.find(candidates->data[i].id); + if (token_iter == token_count.end()) { + continue; + } + + int count = token_iter->second; + candidates->data[i].logit -= float(count) * alpha_frequency + float(count > 0) * alpha_presence; + } + + candidates->sorted = false; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * candidates, const struct llama_grammar * grammar) { + assert(ctx); + const int64_t t_start_sample_us = ggml_time_us(); + + bool allow_eos = false; + for (const auto & stack : grammar->stacks) { + if (stack.empty()) { + allow_eos = true; + break; + } + } + + const llama_token eos = llama_token_eos(); + + std::vector> candidates_decoded; + std::vector candidates_grammar; + + for (size_t i = 0; i < candidates->size; ++i) { + const llama_token id = candidates->data[i].id; + const char * str = llama_token_to_str(ctx, id); + if (id == eos) { + if (!allow_eos) { + candidates->data[i].logit = -INFINITY; + } + } else if (*str == 0) { + candidates->data[i].logit = -INFINITY; + } else { + candidates_decoded.push_back(decode_utf8(str)); + candidates_grammar.push_back({ i, candidates_decoded.back().data() }); + } + } + + const auto rejects = + llama_grammar_reject_candidates(grammar->rules, grammar->stacks, candidates_grammar); + for (auto & reject : rejects) { + candidates->data[reject.index].logit = -INFINITY; + } + + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; +} + +static void llama_log_softmax(float * array, size_t size) { + float max_l = *std::max_element(array, array + size); + float sum = 0.f; + for (size_t i = 0; i < size; ++i) { + float p = expf(array[i] - max_l); + sum += p; + array[i] = p; + } + + for (size_t i = 0; i < size; ++i) { + array[i] = logf(array[i] / sum); + } +} + +void llama_sample_classifier_free_guidance( + struct llama_context * ctx, + llama_token_data_array * candidates, + struct llama_context * guidance_ctx, + float scale) { + int64_t t_start_sample_us = ggml_time_us(); + + assert(ctx); + auto n_vocab = llama_n_vocab(ctx); + assert(n_vocab == (int)candidates->size); + assert(!candidates->sorted); + + std::vector logits_base; + logits_base.reserve(candidates->size); + for (size_t i = 0; i < candidates->size; ++i) { + logits_base.push_back(candidates->data[i].logit); + } + llama_log_softmax(logits_base.data(), candidates->size); + + float* logits_guidance = llama_get_logits(guidance_ctx); + llama_log_softmax(logits_guidance, n_vocab); + + for (int i = 0; i < n_vocab; ++i) { + float logit_guidance = logits_guidance[i]; + float logit_base = logits_base[i]; + candidates->data[i].logit = scale * (logit_base - logit_guidance) + logit_guidance; + } + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } +} + +llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int m, float * mu) { + assert(ctx); + auto N = float(llama_n_vocab(ctx)); + int64_t t_start_sample_us; + t_start_sample_us = ggml_time_us(); + + llama_sample_softmax(nullptr, candidates); + + // Estimate s_hat using the most probable m tokens + float s_hat = 0.0; + float sum_ti_bi = 0.0; + float sum_ti_sq = 0.0; + for (size_t i = 0; i < size_t(m - 1) && i < candidates->size - 1; ++i) { + float t_i = logf(float(i + 2) / float(i + 1)); + float b_i = logf(candidates->data[i].p / candidates->data[i + 1].p); + sum_ti_bi += t_i * b_i; + sum_ti_sq += t_i * t_i; + } + s_hat = sum_ti_bi / sum_ti_sq; + + // Compute k from the estimated s_hat and target surprise value + float epsilon_hat = s_hat - 1; + float k = powf((epsilon_hat * powf(2, *mu)) / (1 - powf(N, -epsilon_hat)), 1 / s_hat); + + // Sample the next word X using top-k sampling + llama_sample_top_k(nullptr, candidates, int(k), 1); + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } + llama_token X = llama_sample_token(ctx, candidates); + t_start_sample_us = ggml_time_us(); + + // Compute error as the difference between observed surprise and target surprise value + size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) { + return candidate.id == X; + })); + float observed_surprise = -log2f(candidates->data[X_idx].p); + float e = observed_surprise - tau; + + // Update mu using the learning rate and error + *mu = *mu - eta * e; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } + return X; +} + +llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu) { + int64_t t_start_sample_us; + t_start_sample_us = ggml_time_us(); + + llama_sample_softmax(ctx, candidates); + + // Truncate the words with surprise values greater than mu + candidates->size = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) { + return -log2f(candidate.p) > *mu; + })); + + if (candidates->size == 0) { + candidates->size = 1; + } + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } + + // Normalize the probabilities of the remaining words + llama_sample_softmax(ctx, candidates); + + // Sample the next word X from the remaining words + llama_token X = llama_sample_token(ctx, candidates); + t_start_sample_us = ggml_time_us(); + + // Compute error as the difference between observed surprise and target surprise value + size_t X_idx = std::distance(candidates->data, std::find_if(candidates->data, candidates->data + candidates->size, [&](const llama_token_data & candidate) { + return candidate.id == X; + })); + float observed_surprise = -log2f(candidates->data[X_idx].p); + float e = observed_surprise - tau; + + // Update mu using the learning rate and error + *mu = *mu - eta * e; + + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + } + return X; +} + +llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates) { + const int64_t t_start_sample_us = ggml_time_us(); + + // Find max element + auto * max_iter = std::max_element(candidates->data, candidates->data + candidates->size, [](const llama_token_data & a, const llama_token_data & b) { + return a.logit < b.logit; + }); + + llama_token result = max_iter->id; + if (ctx) { + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + ctx->n_sample++; + } + return result; +} + +llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates) { + assert(ctx); + const int64_t t_start_sample_us = ggml_time_us(); + llama_sample_softmax(nullptr, candidates); + + std::vector probs; + probs.reserve(candidates->size); + for (size_t i = 0; i < candidates->size; ++i) { + probs.push_back(candidates->data[i].p); + } + + std::discrete_distribution<> dist(probs.begin(), probs.end()); + auto & rng = ctx->rng; + int idx = dist(rng); + + llama_token result = candidates->data[idx].id; + + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; + ctx->n_sample++; + return result; +} + +void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar * grammar, llama_token token) { + const int64_t t_start_sample_us = ggml_time_us(); + + if (token == llama_token_eos()) { + for (const auto & stack : grammar->stacks) { + if (stack.empty()) { + return; + } + } + GGML_ASSERT(false); + } + + const char * str = llama_token_to_str(ctx, token); + // Note terminating 0 in decoded string + auto code_points = decode_utf8(str); + for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { + grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); + } + GGML_ASSERT(!grammar->stacks.empty()); + + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; +} + +// +// quantization +// + +static void llama_convert_tensor_internal(const gguf_load_tensor & tensor, gguf_buffer & output, const int nelements, const int nthread) { + if (output.size < nelements * sizeof(float)) { + output.resize(nelements * sizeof(float)); + } + float * f32_output = (float *) output.addr; + + ggml_type_traits_t qtype; + if (ggml_is_quantized(tensor.type)) { + qtype = ggml_internal_get_type_traits(tensor.type); + if (qtype.to_float == NULL) { + throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor.type))); + } + } else if (tensor.type != GGML_TYPE_F16) { + throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor.type))); + } + + if (nthread < 2) { + if (tensor.type == GGML_TYPE_F16) { + ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor.data, f32_output, nelements); + } else if (ggml_is_quantized(tensor.type)) { + qtype.to_float(tensor.data, f32_output, nelements); + } else { + GGML_ASSERT(false); // unreachable + } + return; + } + + auto block_size = tensor.type == GGML_TYPE_F16 ? 1 : (size_t)ggml_blck_size(tensor.type); + auto block_size_bytes = ggml_type_size(tensor.type); + + GGML_ASSERT(nelements % block_size == 0); + auto nblocks = nelements / block_size; + auto blocks_per_thread = nblocks / nthread; + auto spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count + + std::vector workers; + for (auto tnum = 0, in_buff_offs = 0, out_buff_offs = 0; tnum < nthread; tnum++) { + auto thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread + auto thr_elems = thr_blocks * block_size; // number of elements for this thread + auto thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread + + auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) { + if (typ == GGML_TYPE_F16) { + ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels); + } else { + qtype.to_float(inbuf, outbuf, nels); + } + }; + workers.push_back(std::thread(compute, tensor.type, tensor.data + in_buff_offs, f32_output + out_buff_offs, thr_elems)); + in_buff_offs += thr_block_bytes; + out_buff_offs += thr_elems; + } + for (auto & worker : workers) { + worker.join(); + } + +} + +static void llama_model_quantize_internal(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) { + ggml_type quantized_type; + llama_ftype ftype = params->ftype; + int nthread = params->nthread; + + switch (params->ftype) { + case LLAMA_FTYPE_MOSTLY_Q4_0: quantized_type = GGML_TYPE_Q4_0; break; + case LLAMA_FTYPE_MOSTLY_Q4_1: quantized_type = GGML_TYPE_Q4_1; break; + case LLAMA_FTYPE_MOSTLY_Q5_0: quantized_type = GGML_TYPE_Q5_0; break; + case LLAMA_FTYPE_MOSTLY_Q5_1: quantized_type = GGML_TYPE_Q5_1; break; + case LLAMA_FTYPE_MOSTLY_Q8_0: quantized_type = GGML_TYPE_Q8_0; break; + case LLAMA_FTYPE_MOSTLY_F16: quantized_type = GGML_TYPE_F16; break; + case LLAMA_FTYPE_ALL_F32: quantized_type = GGML_TYPE_F32; break; + +#ifdef GGML_USE_K_QUANTS + // K-quants + case LLAMA_FTYPE_MOSTLY_Q2_K: quantized_type = GGML_TYPE_Q2_K; break; + case LLAMA_FTYPE_MOSTLY_Q3_K_S: + case LLAMA_FTYPE_MOSTLY_Q3_K_M: + case LLAMA_FTYPE_MOSTLY_Q3_K_L: quantized_type = GGML_TYPE_Q3_K; break; + case LLAMA_FTYPE_MOSTLY_Q4_K_S: + case LLAMA_FTYPE_MOSTLY_Q4_K_M: quantized_type = GGML_TYPE_Q4_K; break; + case LLAMA_FTYPE_MOSTLY_Q5_K_S: + case LLAMA_FTYPE_MOSTLY_Q5_K_M: quantized_type = GGML_TYPE_Q5_K; break; + case LLAMA_FTYPE_MOSTLY_Q6_K: quantized_type = GGML_TYPE_Q6_K; break; +#endif + default: throw std::runtime_error(format("invalid output file type %d\n", ftype)); + } + + if (nthread <= 0) { + nthread = std::thread::hardware_concurrency(); + } + + std::unique_ptr model_loader(new llama_model_loader(fname_inp, /*use_mmap*/ false)); + gguf_file_saver file_saver(fname_out.c_str(), model_loader->file_loader.get(), params->ftype); + +#ifdef GGML_USE_K_QUANTS + int n_attention_wv = 0; + int n_feed_forward_w2 = 0; + for (auto& tensor : model_loader->tensors_map.tensors) { + if (tensor.name.find("attention.wv.weight") != std::string::npos) { + ++n_attention_wv; + } + else if (tensor.name.find("feed_forward.w2.weight") != std::string::npos) { + ++n_feed_forward_w2; + } + } + + int i_attention_wv = 0; + int i_feed_forward_w2 = 0; +#endif + + size_t total_size_org = 0; + size_t total_size_new = 0; + std::vector hist_all(1 << 4, 0); + + std::vector workers; + std::mutex mutex; + + auto use_more_bits = [] (int i_layer, int num_layers) -> bool { + return i_layer < num_layers/8 || i_layer >= 7*num_layers/8 || (i_layer - num_layers/8)%3 == 2; + }; + + size_t idx = 0; + for (gguf_load_tensor & tensor : model_loader->tensors_map.tensors) { + gguf_buffer read_data; + read_data.resize(tensor.size); + tensor.data = read_data.addr; + model_loader->load_data_for(tensor); + + printf("[%4zu/%4zu] %36s - %16s, type = %6s, ", + ++idx, model_loader->tensors_map.tensors.size(), + tensor.name.c_str(), llama_format_tensor_shape(tensor.ne).c_str(), + ggml_type_name(tensor.type)); + + // This used to be a regex, but has an extreme cost to compile times. + bool quantize = tensor.name.rfind("weight") == tensor.name.size() - 6; // ends with 'weight'? + + // quantize only 2D tensors + quantize &= (tensor.ne.size() == 2); + quantize &= params->quantize_output_tensor || tensor.name != "output.weight"; + quantize &= quantized_type != tensor.type; + + enum ggml_type new_type; + void * new_data; + size_t new_size; + gguf_buffer work; + + if (!quantize) { + new_type = tensor.type; + new_data = tensor.data; + new_size = tensor.size; + printf("size = %8.3f MB\n", tensor.size/1024.0/1024.0); + } else { + new_type = quantized_type; +#ifdef GGML_USE_K_QUANTS + if (tensor.name == "output.weight") { + int nx = tensor.ne.at(0); + int ny = tensor.ne.at(1); + if (nx % QK_K == 0 && ny % QK_K == 0) { + new_type = GGML_TYPE_Q6_K; + } + } else if (tensor.name.find("attention.wv.weight") != std::string::npos) { + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; + else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) && + use_more_bits(i_attention_wv, n_attention_wv)) new_type = GGML_TYPE_Q6_K; + else if (QK_K == 64 && (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S) && + (i_attention_wv < n_attention_wv/8 || i_attention_wv >= 7*n_attention_wv/8)) new_type = GGML_TYPE_Q6_K; + ++i_attention_wv; + } else if (tensor.name.find("feed_forward.w2.weight") != std::string::npos) { + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; + else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) && + use_more_bits(i_feed_forward_w2, n_feed_forward_w2)) new_type = GGML_TYPE_Q6_K; + //else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && i_feed_forward_w2 < n_feed_forward_w2/8) new_type = GGML_TYPE_Q6_K; + ++i_feed_forward_w2; + } else if (tensor.name.find("attention.wo.weight") != std::string::npos) { + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; + } + bool convert_incompatible_tensor = false; + if (new_type == GGML_TYPE_Q2_K || new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K || + new_type == GGML_TYPE_Q5_K || new_type == GGML_TYPE_Q6_K) { + int nx = tensor.ne.at(0); + int ny = tensor.ne.at(1); + if (nx % QK_K != 0 || ny % QK_K != 0) { + fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K); + convert_incompatible_tensor = true; + } + } + if (convert_incompatible_tensor) { + if (tensor.name == "output.weight") { + new_type = GGML_TYPE_F16; //fall back to F16 instead of just failing. + fprintf(stderr, "F16 will be used for this tensor instead.\n"); + } else if (tensor.name == "tok_embeddings.weight") { + new_type = GGML_TYPE_Q4_0; //fall back to Q4_0 instead of just failing. + fprintf(stderr, "Q4_0 will be used for this tensor instead.\n"); + } else { + throw std::runtime_error("Unsupported tensor size encountered\n"); + } + } +#endif + + float * f32_data; + size_t nelements = tensor.ne.at(0) * tensor.ne.at(1); + gguf_buffer f32_conv_buf; + + if (tensor.type == GGML_TYPE_F32) { + f32_data = (float *) tensor.data; + } else if (ggml_is_quantized(tensor.type) && !params->allow_requantize) { + throw std::runtime_error(format("requantizing from type %s is disabled", ggml_type_name(tensor.type))); + } else { + llama_convert_tensor_internal(tensor, f32_conv_buf, nelements, nthread); + f32_data = (float *) f32_conv_buf.addr; + } + + printf("quantizing to %s .. ", ggml_type_name(new_type)); + fflush(stdout); + + work.resize(nelements * 4); // upper bound on size + new_data = work.addr; + std::vector hist_cur(1 << 4, 0); + + int chunk_size = 32 * 512; + const int nchunk = (nelements + chunk_size - 1)/chunk_size; + const int nthread_use = nthread > 1 ? std::max(1, std::min(nthread, nchunk)) : 1; + if (nthread_use < 2) { + new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nelements, hist_cur.data()); + } else { + size_t counter = 0; + new_size = 0; + auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, nelements, chunk_size] () { + std::vector local_hist; + size_t local_size = 0; + while (true) { + std::unique_lock lock(mutex); + size_t first = counter; counter += chunk_size; + if (first >= nelements) { + if (!local_hist.empty()) { + for (int j=0; j %8.2f MB | hist: ", tensor.size/1024.0/1024.0, new_size/1024.0/1024.0); + int64_t tot_count = 0; + for (size_t i = 0; i < hist_cur.size(); i++) { + hist_all[i] += hist_cur[i]; + tot_count += hist_cur[i]; + } + + if (tot_count > 0) { + for (size_t i = 0; i < hist_cur.size(); i++) { + printf("%5.3f ", hist_cur[i] / float(nelements)); + } + } + printf("\n"); + } + total_size_org += tensor.size; + total_size_new += new_size; + file_saver.write_tensor(tensor, new_type, new_data, new_size); + } + + printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0); + printf("%s: quant size = %8.2f MB\n", __func__, total_size_new/1024.0/1024.0); + + { + int64_t sum_all = 0; + for (size_t i = 0; i < hist_all.size(); i++) { + sum_all += hist_all[i]; + } + + if (sum_all > 0) { + printf("%s: hist: ", __func__); + for (size_t i = 0; i < hist_all.size(); i++) { + printf("%5.3f ", hist_all[i] / float(sum_all)); + } + printf("\n"); + } + } +} + + + +// +// interface implementation +// + +struct llama_model * llama_load_model_from_file( + const char * path_model, + struct llama_context_params params) { + ggml_time_init(); + + llama_model * model = new llama_model; + + ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32; + + if (!llama_model_load(path_model, *model, model->vocab, params.n_ctx, params.n_batch, params.n_gqa, params.rms_norm_eps, params.n_gpu_layers, + params.main_gpu, params.tensor_split, params.rope_freq_base, params.rope_freq_scale,params.low_vram, + memory_type, params.use_mmap, params.use_mlock, params.vocab_only, params.progress_callback, + params.progress_callback_user_data)) { + delete model; + fprintf(stderr, "%s: failed to load model\n", __func__); + return nullptr; + } + + return model; +} + +void llama_free_model(struct llama_model * model) { + delete model; +} + +struct llama_context * llama_new_context_with_model( + struct llama_model * model, + struct llama_context_params params) { + + if (!model) { + return nullptr; + } + + llama_context * ctx = new llama_context(*model); + + if (params.seed == LLAMA_DEFAULT_SEED) { + params.seed = time(NULL); + } + + unsigned cur_percentage = 0; + if (params.progress_callback == NULL) { + params.progress_callback_user_data = &cur_percentage; + params.progress_callback = [](float progress, void * ctx) { + unsigned * cur_percentage_p = (unsigned *) ctx; + unsigned percentage = (unsigned) (100 * progress); + while (percentage > *cur_percentage_p) { + *cur_percentage_p = percentage; + fprintf(stderr, "."); + fflush(stderr); + if (percentage >= 100) { + fprintf(stderr, "\n"); + } + } + }; + } + + ctx->rng = std::mt19937(params.seed); + ctx->logits_all = params.logits_all; + + ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32; + + // reserve memory for context buffers + if (!params.vocab_only) { + if (!kv_cache_init(ctx->model.hparams, ctx->kv_self, memory_type, ctx->model.hparams.n_ctx, params.n_gpu_layers)) { + fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__); + llama_free(ctx); + return nullptr; + } + + { + const size_t memory_size = ggml_nbytes(ctx->kv_self.k) + ggml_nbytes(ctx->kv_self.v); + fprintf(stderr, "%s: kv self size = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0); + } + + const auto & hparams = ctx->model.hparams; + + // resized during inference + if (params.logits_all) { + ctx->logits.reserve(hparams.n_ctx*hparams.n_vocab); + } else { + ctx->logits.reserve(hparams.n_vocab); + } + + if (params.embedding){ + ctx->embedding.resize(hparams.n_embd); + } + + ctx->buf_compute.resize(MEM_REQ_EVAL().at(ctx->model.type) + ggml_graph_overhead()); + + ctx->buf_scratch[0].resize(MEM_REQ_SCRATCH0(hparams.n_ctx).at(ctx->model.type)); + ctx->buf_scratch[1].resize(MEM_REQ_SCRATCH1().at(ctx->model.type)); + } + +#ifdef GGML_USE_METAL + if (params.n_gpu_layers > 0) { + // this allocates all Metal resources and memory buffers + ctx->ctx_metal = ggml_metal_init(1); + + void * data_ptr = NULL; + size_t data_size = 0; + + if (params.use_mmap) { + data_ptr = ctx->model.mapping->addr; + data_size = ctx->model.mapping->size; + } else { + data_ptr = ggml_get_mem_buffer(ctx->model.ctx); + data_size = ggml_get_mem_size (ctx->model.ctx); + } + + const size_t max_size = ggml_get_max_tensor_size(ctx->model.ctx); + + fprintf(stderr, "%s: max tensor size = %8.2f MB\n", __func__, max_size/1024.0/1024.0); + +#define LLAMA_METAL_CHECK_BUF(result) \ + if (!(result)) { \ + fprintf(stderr, "%s: failed to add buffer\n", __func__); \ + llama_free(ctx); \ + return NULL; \ + } + + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "data", data_ptr, data_size, max_size)); + + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "eval", ctx->buf_compute.addr, ctx->buf_compute.size, 0)); + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "kv", ctx->kv_self.buf.addr, ctx->kv_self.buf.size, 0)); + + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr0", ctx->buf_scratch[0].addr, ctx->buf_scratch[0].size, 0)); + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr1", ctx->buf_scratch[1].addr, ctx->buf_scratch[1].size, 0)); +#undef LLAMA_METAL_CHECK_BUF + } +#endif + +#ifdef GGML_USE_MPI + ctx->ctx_mpi = ggml_mpi_init(); + + if (ggml_mpi_rank(ctx->ctx_mpi) > 0) { + // Enter a blocking eval loop with dummy input, letting rank=0 drive the process + const std::vector tmp(ctx->model.hparams.n_ctx, llama_token_bos()); + while (!llama_eval(ctx, tmp.data(), tmp.size(), 0, 0)) {}; + llama_backend_free(); + exit(1); + } +#endif + + return ctx; +} + +struct llama_context * llama_init_from_file( + const char * path_model, + struct llama_context_params params) { + + struct llama_model * model = llama_load_model_from_file(path_model, params); + if (!model) { + return nullptr; + } + struct llama_context * ctx = llama_new_context_with_model(model, params); + ctx->model_owner = true; + return ctx; +} + +void llama_free(struct llama_context * ctx) { + if (ctx->model_owner) { + delete &ctx->model; + } + delete ctx; +} + +int llama_model_quantize( + const char * fname_inp, + const char * fname_out, + const llama_model_quantize_params *params) { + try { + llama_model_quantize_internal(fname_inp, fname_out, params); + return 0; + } catch (const std::exception & err) { + fprintf(stderr, "%s: failed to quantize: %s\n", __func__, err.what()); + return 1; + } +} + +int llama_apply_lora_from_file_internal(const struct llama_model & model, const char * path_lora, const char * path_base_model, int n_threads) { + fprintf(stderr, "%s: applying lora adapter from '%s' - please wait ...\n", __func__, path_lora); + + const int64_t t_start_lora_us = ggml_time_us(); + + auto fin = std::ifstream(path_lora, std::ios::binary); + if (!fin) { + fprintf(stderr, "%s: failed to open '%s'\n", __func__, path_lora); + return 1; + } + + // verify magic and version + { + uint32_t magic; + fin.read((char *) &magic, sizeof(magic)); + uint32_t format_version; + fin.read((char *) &format_version, sizeof(format_version)); + + if (format_version != 1) { + fprintf(stderr, "%s: unsupported file version\n", __func__ ); + return 1; + } + } + + int32_t lora_r; + int32_t lora_alpha; + fin.read((char *) &lora_r, sizeof(lora_r)); + fin.read((char *) &lora_alpha, sizeof(lora_alpha)); + float scaling = (float)lora_alpha / (float)lora_r; + + fprintf(stderr, "%s: r = %d, alpha = %d, scaling = %.2f\n", __func__, lora_r, lora_alpha, scaling); + + + // create a temporary ggml context to store the lora tensors + // todo: calculate size from biggest possible tensor + std::vector lora_buf(1024ull * 1024ull * 1024ull); + struct ggml_init_params params; + params.mem_size = lora_buf.size(); + params.mem_buffer = lora_buf.data(); + params.no_alloc = false; + + ggml_context * lora_ctx = ggml_init(params); + std::unordered_map lora_tensors; + + // create a name -> tensor map of the model to accelerate lookups + std::unordered_map model_tensors; + for (const auto & kv: model.tensors_by_name) { + model_tensors.insert(kv); + } + + + // load base model + std::unique_ptr model_loader; + ggml_context * base_ctx = NULL; + gguf_buffer base_buf; + if (path_base_model) { + fprintf(stderr, "%s: loading base model from '%s'\n", __func__, path_base_model); + model_loader.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true)); + + size_t ctx_size; + size_t mmapped_size; + model_loader->calc_sizes(&ctx_size, &mmapped_size); + base_buf.resize(ctx_size); + + ggml_init_params base_params; + base_params.mem_size = base_buf.size; + base_params.mem_buffer = base_buf.addr; + base_params.no_alloc = model_loader->use_mmap; + + base_ctx = ggml_init(base_params); + + model_loader->ggml_ctx = base_ctx; + + // maybe this should in llama_model_loader + if (model_loader->use_mmap) { + model_loader->mapping.reset(new gguf_mmap(&model_loader->file_loader->file, /* prefetch */ 0, ggml_is_numa())); + } + } + + // read tensors and apply + bool warned = false; + int n_tensors = 0; + + std::vector work_buffer; + + while (true) { + int32_t n_dims; + int32_t length; + int32_t ftype; + + fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); + fin.read(reinterpret_cast(&length), sizeof(length)); + fin.read(reinterpret_cast(&ftype), sizeof(ftype)); + if (fin.eof()) { + break; + } + + int32_t ne[2] = { 1, 1 }; + for (int i = 0; i < n_dims; ++i) { + fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i])); + } + + std::string name; + { + char buf[1024]; + fin.read(buf, length); + name = std::string(buf, length); + } + + // check for lora suffix and get the type of tensor + const std::string lora_suffix = ".lora"; + size_t pos = name.rfind(lora_suffix); + if (pos == std::string::npos) { + fprintf(stderr, "%s: error: '%s' is not a lora tensor\n", __func__, name.c_str()); + return 1; + } + + std::string lora_type = name.substr(pos + lora_suffix.length()); + std::string base_name = name; + base_name.erase(pos); + // fprintf(stderr, "%s: %s => %s (lora type %s) ", __func__, name.c_str(),base_name.c_str(), lora_type.c_str()); + + if (model_tensors.find(base_name) == model_tensors.end()) { + fprintf(stderr, "%s: unknown tensor '%s' in lora adapter\n", __func__, name.data()); + return 1; + } + + // create ggml tensor + ggml_type wtype; + switch (ftype) { + case 0: wtype = GGML_TYPE_F32; break; + case 1: wtype = GGML_TYPE_F16; break; + default: + { + fprintf(stderr, "%s: invalid tensor data type '%d'\n", + __func__, ftype); + return false; + } + } + ggml_tensor * lora_tensor; + if (n_dims == 2) { + lora_tensor = ggml_new_tensor_2d(lora_ctx, wtype, ne[0], ne[1]); + } + else { + fprintf(stderr, "%s: unsupported tensor dimension %d\n", __func__, n_dims); + return 1; + } + ggml_set_name(lora_tensor, "lora_tensor"); + + // load tensor data + size_t offset = fin.tellg(); + size_t tensor_data_size = ggml_nbytes(lora_tensor); + offset = (offset + 31) & -32; + fin.seekg(offset); + fin.read((char*)lora_tensor->data, tensor_data_size); + + lora_tensors[name] = lora_tensor; + + // check if we have both A and B tensors and apply + if (lora_tensors.find(base_name + ".loraA") != lora_tensors.end() && + lora_tensors.find(base_name + ".loraB") != lora_tensors.end()) { + + ggml_tensor * dest_t = model_tensors[base_name]; + + offload_func_t offload_func = llama_nop; + offload_func_t offload_func_force_inplace = llama_nop; + +#ifdef GGML_USE_CUBLAS + if (dest_t->backend == GGML_BACKEND_GPU || dest_t->backend == GGML_BACKEND_GPU_SPLIT) { + if (dest_t->type != GGML_TYPE_F16) { + throw std::runtime_error(format( + "%s: error: the simultaneous use of LoRAs and GPU acceleration is only supported for f16 models", __func__)); + } + offload_func = ggml_cuda_assign_buffers; + offload_func_force_inplace = ggml_cuda_assign_buffers_force_inplace; + } +#endif // GGML_USE_CUBLAS + + ggml_tensor * base_t; + if (model_loader) { + // load from base model + if (model_loader->tensors_map.name_to_idx.find(base_name) == model_loader->tensors_map.name_to_idx.end()) { + fprintf(stderr, "%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str()); + return 1; + } + size_t idx = model_loader->tensors_map.name_to_idx[base_name]; + gguf_load_tensor & lt = model_loader->tensors_map.tensors[idx]; + base_t = model_loader->get_tensor(base_name, { (uint32_t)dest_t->ne[0], (uint32_t)dest_t->ne[1] }, GGML_BACKEND_CPU); + lt.data = (uint8_t *) lt.ggml_tensor->data; + model_loader->load_data_for(lt); + lt.ggml_tensor->data = lt.data; + } + else { + base_t = dest_t; + } + + if (ggml_is_quantized(base_t->type)) { + if (!warned) { + fprintf(stderr, "%s: warning: using a lora adapter with a quantized model may result in poor quality, " + "use a f16 or f32 base model with --lora-base\n", __func__); + warned = true; + } + } + + ggml_tensor * loraA = lora_tensors[base_name + ".loraA"]; + GGML_ASSERT(loraA->type == GGML_TYPE_F32); + ggml_set_name(loraA, "loraA"); + + ggml_tensor * loraB = lora_tensors[base_name + ".loraB"]; + GGML_ASSERT(loraB->type == GGML_TYPE_F32); + ggml_set_name(loraB, "loraB"); + + if (base_t->ne[0] != loraA->ne[1] || base_t->ne[1] != loraB->ne[1]) { + fprintf(stderr, "%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");" + " are you sure that this adapter is for this model?\n", __func__, base_t->ne[0], loraA->ne[1]); + return 1; + } + + // w = w + BA*s + ggml_tensor * BA = ggml_mul_mat(lora_ctx, loraA, loraB); + offload_func(BA); + ggml_set_name(BA, "BA"); + + if (scaling != 1.0f) { + ggml_tensor * scale_tensor = ggml_new_f32(lora_ctx, scaling); + ggml_set_name(scale_tensor, "scale_tensor"); + + BA = ggml_scale_inplace(lora_ctx, BA, scale_tensor); + offload_func(BA); + ggml_set_name(BA, "BA_scaled"); + } + + ggml_tensor * r; + if (base_t == dest_t) { + r = ggml_add_inplace(lora_ctx, dest_t, BA); + offload_func_force_inplace(r); + ggml_set_name(r, "r_add_inplace"); + } + else { + r = ggml_add(lora_ctx, base_t, BA); + offload_func(r); + ggml_set_name(r, "r_add"); + + r = ggml_cpy(lora_ctx, r, dest_t); + offload_func(r); + ggml_set_name(r, "r_cpy"); + } + + struct ggml_cgraph gf = ggml_build_forward(r); + + ggml_graph_compute_helper(work_buffer, &gf, n_threads); + + // we won't need these tensors again, reset the context to save memory + ggml_free(lora_ctx); + lora_ctx = ggml_init(params); + lora_tensors.clear(); + + n_tensors++; + if (n_tensors % 4 == 0) { + fprintf(stderr, "."); + } + } + } + + // TODO: this should be in a destructor, it will leak on failure + ggml_free(lora_ctx); + if (base_ctx) { + ggml_free(base_ctx); + } + + const int64_t t_lora_us = ggml_time_us() - t_start_lora_us; + fprintf(stderr, " done (%.2f ms)\n", t_lora_us / 1000.0); + + return 0; +} + +int llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lora, const char * path_base_model, int n_threads) { + try { + return llama_apply_lora_from_file_internal(ctx->model, path_lora, path_base_model, n_threads); + } catch (const std::exception & err) { + fprintf(stderr, "%s: failed to apply lora adapter: %s\n", __func__, err.what()); + return 1; + } +} + +int llama_model_apply_lora_from_file(const struct llama_model * model, const char * path_lora, const char * path_base_model, int n_threads) { + try { + return llama_apply_lora_from_file_internal(*model, path_lora, path_base_model, n_threads); + } catch (const std::exception & err) { + fprintf(stderr, "%s: failed to apply lora adapter: %s\n", __func__, err.what()); + return 1; + } +} + +int llama_get_kv_cache_token_count(const struct llama_context * ctx) { + return ctx->kv_self.n; +} + +#define LLAMA_MAX_RNG_STATE (64*1024) + +void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed) { + if (seed == LLAMA_DEFAULT_SEED) { + seed = time(NULL); + } + ctx->rng.seed(seed); +} + +// Returns the *maximum* size of the state +size_t llama_get_state_size(const struct llama_context * ctx) { + // we don't know size of rng until we actually serialize it. so reserve more than enough memory for its serialized state. + // for reference, std::mt19937(1337) serializes to 6701 bytes. + const size_t s_rng_size = sizeof(size_t); + const size_t s_rng = LLAMA_MAX_RNG_STATE; + const size_t s_logits_capacity = sizeof(size_t); + const size_t s_logits_size = sizeof(size_t); + const size_t s_logits = ctx->logits.capacity() * sizeof(float); + const size_t s_embedding_size = sizeof(size_t); + const size_t s_embedding = ctx->embedding.size() * sizeof(float); + const size_t s_kv_size = sizeof(size_t); + const size_t s_kv_ntok = sizeof(int); + const size_t s_kv = ctx->kv_self.buf.size; + + const size_t s_total = ( + + s_rng_size + + s_rng + + s_logits_capacity + + s_logits_size + + s_logits + + s_embedding_size + + s_embedding + + s_kv_size + + s_kv_ntok + + s_kv + ); + + return s_total; +} + +// Copies the state to the specified destination address +size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { + uint8_t * out = dst; + + // copy rng + { + std::stringstream rng_ss; + rng_ss << ctx->rng; + + const size_t rng_size = rng_ss.str().size(); + char rng_buf[LLAMA_MAX_RNG_STATE]; + + memset(&rng_buf[0], 0, LLAMA_MAX_RNG_STATE); + memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size()); + + memcpy(out, &rng_size, sizeof(rng_size)); out += sizeof(rng_size); + memcpy(out, &rng_buf[0], LLAMA_MAX_RNG_STATE); out += LLAMA_MAX_RNG_STATE; + } + + // copy logits + { + const size_t logits_cap = ctx->logits.capacity(); + const size_t logits_size = ctx->logits.size(); + + memcpy(out, &logits_cap, sizeof(logits_cap)); out += sizeof(logits_cap); + memcpy(out, &logits_size, sizeof(logits_size)); out += sizeof(logits_size); + + if (logits_size) { + memcpy(out, ctx->logits.data(), logits_size * sizeof(float)); + } + + out += logits_cap * sizeof(float); + } + + // copy embeddings + { + const size_t embedding_size = ctx->embedding.size(); + + memcpy(out, &embedding_size, sizeof(embedding_size)); out += sizeof(embedding_size); + + if (embedding_size) { + memcpy(out, ctx->embedding.data(), embedding_size * sizeof(float)); + out += embedding_size * sizeof(float); + } + } + + // copy kv cache + { + const auto & kv_self = ctx->kv_self; + const auto & hparams = ctx->model.hparams; + const int n_layer = hparams.n_layer; + const int n_embd = hparams.n_embd; + const int n_ctx = hparams.n_ctx; + + const size_t kv_size = kv_self.buf.size; + const int kv_ntok = llama_get_kv_cache_token_count(ctx); + + memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size); + memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok); + + if (kv_size) { + const size_t elt_size = ggml_element_size(kv_self.k); + + ggml_context * cpy_ctx = ggml_init({ 4096, NULL, /* no_alloc */ true }); + ggml_cgraph gf{}; + + ggml_tensor * kout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_ntok, n_layer); + kout3d->data = out; + out += ggml_nbytes(kout3d); + + ggml_tensor * vout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_ntok, n_embd, n_layer); + vout3d->data = out; + out += ggml_nbytes(vout3d); + + ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k, + n_embd, kv_ntok, n_layer, + elt_size*n_embd, elt_size*n_embd*n_ctx, 0); + + ggml_tensor * v3d = ggml_view_3d(cpy_ctx, kv_self.v, + kv_ntok, n_embd, n_layer, + elt_size*n_ctx, elt_size*n_ctx*n_embd, 0); + + ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, k3d, kout3d)); + ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, v3d, vout3d)); + ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1); + + ggml_free(cpy_ctx); + } + } + + const size_t written = out - dst; + const size_t max_size = llama_get_state_size(ctx); + + GGML_ASSERT(written <= max_size); + + return written; +} + +// Sets the state reading from the specified source address +size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { + uint8_t * inp = src; + + // set rng + { + size_t rng_size; + char rng_buf[LLAMA_MAX_RNG_STATE]; + + memcpy(&rng_size, inp, sizeof(rng_size)); inp += sizeof(rng_size); + memcpy(&rng_buf[0], inp, LLAMA_MAX_RNG_STATE); inp += LLAMA_MAX_RNG_STATE; + + std::stringstream rng_ss; + rng_ss.str(std::string(&rng_buf[0], rng_size)); + rng_ss >> ctx->rng; + + GGML_ASSERT(rng_ss.fail() == false); + } + + // set logits + { + size_t logits_cap; + size_t logits_size; + + memcpy(&logits_cap, inp, sizeof(logits_cap)); inp += sizeof(logits_cap); + memcpy(&logits_size, inp, sizeof(logits_size)); inp += sizeof(logits_size); + + GGML_ASSERT(ctx->logits.capacity() == logits_cap); + + if (logits_size) { + ctx->logits.resize(logits_size); + memcpy(ctx->logits.data(), inp, logits_size * sizeof(float)); + } + + inp += logits_cap * sizeof(float); + } + + // set embeddings + { + size_t embedding_size; + + memcpy(&embedding_size, inp, sizeof(embedding_size)); inp += sizeof(embedding_size); + + GGML_ASSERT(ctx->embedding.capacity() == embedding_size); + + if (embedding_size) { + memcpy(ctx->embedding.data(), inp, embedding_size * sizeof(float)); + inp += embedding_size * sizeof(float); + } + } + + // set kv cache + { + const auto & kv_self = ctx->kv_self; + const auto & hparams = ctx->model.hparams; + const int n_layer = hparams.n_layer; + const int n_embd = hparams.n_embd; + const int n_ctx = hparams.n_ctx; + + size_t kv_size; + int kv_ntok; + + memcpy(&kv_size, inp, sizeof(kv_size)); inp += sizeof(kv_size); + memcpy(&kv_ntok, inp, sizeof(kv_ntok)); inp += sizeof(kv_ntok); + + if (kv_size) { + GGML_ASSERT(kv_self.buf.size == kv_size); + + const size_t elt_size = ggml_element_size(kv_self.k); + + ggml_context * cpy_ctx = ggml_init({ 4096, NULL, /* no_alloc */ true }); + ggml_cgraph gf{}; + + ggml_tensor * kin3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_ntok, n_layer); + kin3d->data = (void *) inp; + inp += ggml_nbytes(kin3d); + + ggml_tensor * vin3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_ntok, n_embd, n_layer); + vin3d->data = (void *) inp; + inp += ggml_nbytes(vin3d); + + ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k, + n_embd, kv_ntok, n_layer, + elt_size*n_embd, elt_size*n_embd*n_ctx, 0); + + ggml_tensor * v3d = ggml_view_3d(cpy_ctx, kv_self.v, + kv_ntok, n_embd, n_layer, + elt_size*n_ctx, elt_size*n_ctx*n_embd, 0); + + ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, kin3d, k3d)); + ggml_build_forward_expand(&gf, ggml_cpy(cpy_ctx, vin3d, v3d)); + ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1); + + ggml_free(cpy_ctx); + } + + ctx->kv_self.n = kv_ntok; + } + + const size_t nread = inp - src; + const size_t max_size = llama_get_state_size(ctx); + + GGML_ASSERT(nread <= max_size); + + return nread; +} + +static bool llama_load_session_file_internal(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { + gguf_file file(path_session, "rb"); + GGML_UNUSED(ctx); + GGML_UNUSED(path_session); + GGML_UNUSED(tokens_out); + GGML_UNUSED(n_token_capacity); + GGML_UNUSED(n_token_count_out); + + +// TODO: implement with GGUF format + return true; +} + +bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { + try { + return llama_load_session_file_internal(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out); + } catch (const std::exception & err) { + fprintf(stderr, "error loading session file: %s\n", err.what()); + return false; + } +} + +bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { + gguf_file file(path_session, "wb"); + GGML_UNUSED(ctx); + GGML_UNUSED(tokens); + GGML_UNUSED(n_token_count); + + // TODO: implement with GGUF format + + return true; +} + +int llama_eval( + struct llama_context * ctx, + const llama_token * tokens, + int n_tokens, + int n_past, + int n_threads) { + if (!llama_eval_internal(*ctx, tokens, nullptr, n_tokens, n_past, n_threads, nullptr)) { + fprintf(stderr, "%s: failed to eval\n", __func__); + return 1; + } + + // get a more accurate load time, upon first eval + // TODO: fix this + if (!ctx->has_evaluated_once) { + ctx->t_load_us = ggml_time_us() - ctx->t_start_us; + ctx->has_evaluated_once = true; + } + + return 0; +} + + +int llama_eval_embd( + struct llama_context * ctx, + const float * embd, + int n_tokens, + int n_past, + int n_threads) { + if (!llama_eval_internal(*ctx, nullptr, embd, n_tokens, n_past, n_threads, nullptr)) { + fprintf(stderr, "%s: failed to eval\n", __func__); + return 1; + } + + // get a more accurate load time, upon first eval + // TODO: fix this + if (!ctx->has_evaluated_once) { + ctx->t_load_us = ggml_time_us() - ctx->t_start_us; + ctx->has_evaluated_once = true; + } + + return 0; +} + +int llama_eval_export(struct llama_context * ctx, const char * fname) { + const int n_batch = 1; + const int n_ctx = 512 - n_batch; + + const std::vector tmp(n_batch, llama_token_bos()); + + if (!llama_eval_internal(*ctx, tmp.data(), nullptr, tmp.size(), n_ctx, 1, fname)) { + fprintf(stderr, "%s: failed to eval\n", __func__); + return 1; + } + + return 0; +} + +int llama_tokenize_with_model( + const struct llama_model * model, + const char * text, + llama_token * tokens, + int n_max_tokens, + bool add_bos) { + auto res = llama_tokenize(model->vocab, text, add_bos); + + if (n_max_tokens < (int) res.size()) { + fprintf(stderr, "%s: too many tokens\n", __func__); + return -((int) res.size()); + } + + for (size_t i = 0; i < res.size(); i++) { + tokens[i] = res[i]; + } + + return res.size(); +} + +int llama_tokenize( + struct llama_context * ctx, + const char * text, + llama_token * tokens, + int n_max_tokens, + bool add_bos) { + return llama_tokenize_with_model(&ctx->model, text, tokens, n_max_tokens, add_bos); +} + +int llama_n_vocab_from_model(const struct llama_model * model) { + return model->vocab.id_to_token.size(); +} + +int llama_n_ctx_from_model(const struct llama_model * model) { + return model->hparams.n_ctx; +} + +int llama_n_embd_from_model(const struct llama_model * model) { + return model->hparams.n_embd; +} + +int llama_n_vocab(const struct llama_context * ctx) { + return ctx->model.vocab.id_to_token.size(); +} + +int llama_n_ctx(const struct llama_context * ctx) { + return ctx->model.hparams.n_ctx; +} + +int llama_n_embd(const struct llama_context * ctx) { + return ctx->model.hparams.n_embd; +} + +int llama_get_vocab_from_model( + const struct llama_model * model, + const char * * strings, + float * scores, + int capacity) { + int n = std::min(capacity, (int) model->vocab.id_to_token.size()); + for (int i = 0; ivocab.id_to_token[i].tok.c_str(); + scores[i] = model->vocab.id_to_token[i].score; + } + return n; +} + +int llama_get_vocab( + const struct llama_context * ctx, + const char * * strings, + float * scores, + int capacity) { + return llama_get_vocab_from_model(&ctx->model, strings, scores, capacity); +} + +float * llama_get_logits(struct llama_context * ctx) { + return ctx->logits.data(); +} + +float * llama_get_embeddings(struct llama_context * ctx) { + return ctx->embedding.data(); +} + +const char * llama_token_to_str_with_model(const struct llama_model * model, llama_token token) { + if (token >= llama_n_vocab_from_model(model)) { + return nullptr; + } + + return model->vocab.id_to_token[token].tok.c_str(); +} + +const char * llama_token_to_str(const struct llama_context * ctx, llama_token token) { + return llama_token_to_str_with_model(&ctx->model, token); +} + +llama_token llama_token_bos() { + return 1; +} + +llama_token llama_token_eos() { + return 2; +} + +llama_token llama_token_nl() { + return 13; +} + +struct llama_timings llama_get_timings(struct llama_context * ctx) { + struct llama_timings result = { + /*.t_start_ms =*/ 1e-3 * ctx->t_start_us, + /*.t_end_ms =*/ 1.00 * ggml_time_ms(), + /*.t_load_ms =*/ 1e-3 * ctx->t_load_us, + /*.t_sample_ms =*/ 1e-3 * ctx->t_sample_us, + /*.t_p_eval_ms =*/ 1e-3 * ctx->t_p_eval_us, + /*.t_eval_ms =*/ 1e-3 * ctx->t_eval_us, + + /*.n_sample =*/ std::max(1, ctx->n_sample), + /*.n_p_eval =*/ std::max(1, ctx->n_p_eval), + /*.n_eval =*/ std::max(1, ctx->n_eval), + }; + + return result; +} + +void llama_print_timings(struct llama_context * ctx) { + const llama_timings timings = llama_get_timings(ctx); + + fprintf(stderr, "\n"); + fprintf(stderr, "%s: load time = %8.2f ms\n", __func__, timings.t_load_ms); + fprintf(stderr, "%s: sample time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", + __func__, timings.t_sample_ms, timings.n_sample, timings.t_sample_ms / timings.n_sample, 1e3 / timings.t_sample_ms * timings.n_sample); + fprintf(stderr, "%s: prompt eval time = %8.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", + __func__, timings.t_p_eval_ms, timings.n_p_eval, timings.t_p_eval_ms / timings.n_p_eval, 1e3 / timings.t_p_eval_ms * timings.n_p_eval); + fprintf(stderr, "%s: eval time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", + __func__, timings.t_eval_ms, timings.n_eval, timings.t_eval_ms / timings.n_eval, 1e3 / timings.t_eval_ms * timings.n_eval); + fprintf(stderr, "%s: total time = %8.2f ms\n", __func__, (timings.t_end_ms - timings.t_start_ms)); +} + +void llama_reset_timings(struct llama_context * ctx) { + ctx->t_start_us = ggml_time_us(); + ctx->t_sample_us = ctx->n_sample = 0; + ctx->t_eval_us = ctx->n_eval = 0; + ctx->t_p_eval_us = ctx->n_p_eval = 0; +} + +const char * llama_print_system_info(void) { + static std::string s; + + s = ""; + s += "AVX = " + std::to_string(ggml_cpu_has_avx()) + " | "; + s += "AVX2 = " + std::to_string(ggml_cpu_has_avx2()) + " | "; + s += "AVX512 = " + std::to_string(ggml_cpu_has_avx512()) + " | "; + s += "AVX512_VBMI = " + std::to_string(ggml_cpu_has_avx512_vbmi()) + " | "; + s += "AVX512_VNNI = " + std::to_string(ggml_cpu_has_avx512_vnni()) + " | "; + s += "FMA = " + std::to_string(ggml_cpu_has_fma()) + " | "; + s += "NEON = " + std::to_string(ggml_cpu_has_neon()) + " | "; + s += "ARM_FMA = " + std::to_string(ggml_cpu_has_arm_fma()) + " | "; + s += "F16C = " + std::to_string(ggml_cpu_has_f16c()) + " | "; + s += "FP16_VA = " + std::to_string(ggml_cpu_has_fp16_va()) + " | "; + s += "WASM_SIMD = " + std::to_string(ggml_cpu_has_wasm_simd()) + " | "; + s += "BLAS = " + std::to_string(ggml_cpu_has_blas()) + " | "; + s += "SSE3 = " + std::to_string(ggml_cpu_has_sse3()) + " | "; + s += "VSX = " + std::to_string(ggml_cpu_has_vsx()) + " | "; + + return s.c_str(); +} + +// For internal test use +const std::vector>& llama_internal_get_tensor_map(struct llama_context * ctx) { + return ctx->model.tensors_by_name; +} diff --git a/gguf-llama.h b/gguf-llama.h index d3c0d6b87f437..540167bd196ba 100644 --- a/gguf-llama.h +++ b/gguf-llama.h @@ -1,449 +1,449 @@ -#ifndef LLAMA_H -#define LLAMA_H - -#include "ggml.h" -#ifdef GGML_USE_CUBLAS -#include "ggml-cuda.h" -#define LLAMA_MAX_DEVICES GGML_CUDA_MAX_DEVICES -#else -#define LLAMA_MAX_DEVICES 1 -#endif // GGML_USE_CUBLAS -#include -#include -#include - -#ifdef LLAMA_SHARED -# if defined(_WIN32) && !defined(__MINGW32__) -# ifdef LLAMA_BUILD -# define LLAMA_API __declspec(dllexport) -# else -# define LLAMA_API __declspec(dllimport) -# endif -# else -# define LLAMA_API __attribute__ ((visibility ("default"))) -# endif -#else -# define LLAMA_API -#endif - -#ifdef __GNUC__ -# define DEPRECATED(func, hint) func __attribute__((deprecated(hint))) -#elif defined(_MSC_VER) -# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func -#else -# define DEPRECATED(func, hint) func -#endif - -#define LLAMA_DEFAULT_SEED 0xFFFFFFFF - -#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL) -// Defined when llama.cpp is compiled with support for offloading model layers to GPU. -#define LLAMA_SUPPORTS_GPU_OFFLOAD -#endif - -#ifndef LLAMA_DEFAULT_RMS_EPS -#define LLAMA_DEFAULT_RMS_EPS 5e-6f -#endif - -#ifdef __cplusplus -extern "C" { -#endif - - // - // C interface - // - // TODO: show sample usage - // - - struct llama_model; - struct llama_context; - - typedef int llama_token; - - typedef struct llama_token_data { - llama_token id; // token id - float logit; // log-odds of the token - float p; // probability of the token - } llama_token_data; - - typedef struct llama_token_data_array { - llama_token_data * data; - size_t size; - bool sorted; - } llama_token_data_array; - - typedef void (*llama_progress_callback)(float progress, void *ctx); - - struct llama_context_params { - uint32_t seed; // RNG seed, -1 for random - int32_t n_ctx; // text context - int32_t n_batch; // prompt processing batch size - int32_t n_gqa; // grouped-query attention (TEMP - will be moved to model hparams) - float rms_norm_eps; // rms norm epsilon (TEMP - will be moved to model hparams) - int32_t n_gpu_layers; // number of layers to store in VRAM - int32_t main_gpu; // the GPU that is used for scratch and small tensors - - const float * tensor_split; // how to split layers across multiple GPUs (size: LLAMA_MAX_DEVICES) - - // ref: https://github.com/ggerganov/llama.cpp/pull/2054 - float rope_freq_base; // RoPE base frequency - float rope_freq_scale; // RoPE frequency scaling factor - - // called with a progress value between 0 and 1, pass NULL to disable - llama_progress_callback progress_callback; - // context pointer passed to the progress callback - void * progress_callback_user_data; - - // Keep the booleans together to avoid misalignment during copy-by-value. - bool low_vram; // if true, reduce VRAM usage at the cost of performance - bool f16_kv; // use fp16 for KV cache - bool logits_all; // the llama_eval() call computes all logits, not just the last one - bool vocab_only; // only load the vocabulary, no weights - bool use_mmap; // use mmap if possible - bool use_mlock; // force system to keep model in RAM - bool embedding; // embedding mode only - }; - // model file types - enum llama_ftype { - LLAMA_FTYPE_ALL_F32 = 0, - LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors - LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors - LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors - LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 - // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed - // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed - LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors - LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors - LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors - LLAMA_FTYPE_MOSTLY_Q2_K = 10,// except 1d tensors - LLAMA_FTYPE_MOSTLY_Q3_K_S = 11,// except 1d tensors - LLAMA_FTYPE_MOSTLY_Q3_K_M = 12,// except 1d tensors - LLAMA_FTYPE_MOSTLY_Q3_K_L = 13,// except 1d tensors - LLAMA_FTYPE_MOSTLY_Q4_K_S = 14,// except 1d tensors - LLAMA_FTYPE_MOSTLY_Q4_K_M = 15,// except 1d tensors - LLAMA_FTYPE_MOSTLY_Q5_K_S = 16,// except 1d tensors - LLAMA_FTYPE_MOSTLY_Q5_K_M = 17,// except 1d tensors - LLAMA_FTYPE_MOSTLY_Q6_K = 18,// except 1d tensors - }; - - // model quantization parameters - typedef struct llama_model_quantize_params { - int nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() - enum llama_ftype ftype; // quantize to this llama_ftype - bool allow_requantize; // allow quantizing non-f32/f16 tensors - bool quantize_output_tensor; // quantize output.weight - } llama_model_quantize_params; - - // grammar types - struct llama_grammar; - - // grammar element type - enum llama_gretype { - // end of rule definition - LLAMA_GRETYPE_END = 0, - - // start of alternate definition for rule - LLAMA_GRETYPE_ALT = 1, - - // non-terminal element: reference to rule - LLAMA_GRETYPE_RULE_REF = 2, - - // terminal element: character (code point) - LLAMA_GRETYPE_CHAR = 3, - - // inverse char(s) ([^a], [^a-b] [^abc]) - LLAMA_GRETYPE_CHAR_NOT = 4, - - // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to - // be an inclusive range ([a-z]) - LLAMA_GRETYPE_CHAR_RNG_UPPER = 5, - - // modifies a preceding LLAMA_GRETYPE_CHAR or - // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) - LLAMA_GRETYPE_CHAR_ALT = 6, - }; - - typedef struct llama_grammar_element { - enum llama_gretype type; - uint32_t value; // Unicode code point or rule ID - } llama_grammar_element; - - // performance timing information - struct llama_timings { - double t_start_ms; - double t_end_ms; - double t_load_ms; - double t_sample_ms; - double t_p_eval_ms; - double t_eval_ms; - - int32_t n_sample; - int32_t n_p_eval; - int32_t n_eval; - }; - - LLAMA_API int llama_max_devices(); - - LLAMA_API struct llama_context_params llama_context_default_params(); - LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(); - - LLAMA_API bool llama_mmap_supported(); - LLAMA_API bool llama_mlock_supported(); - - // TODO: not great API - very likely to change - // Initialize the llama + ggml backend - // If numa is true, use NUMA optimizations - // Call once at the start of the program - LLAMA_API void llama_backend_init(bool numa); - // Call once at the end of the program - currently only used for MPI - LLAMA_API void llama_backend_free(); - - LLAMA_API int64_t llama_time_us(); - - LLAMA_API struct llama_model * llama_load_model_from_file( - const char * path_model, - struct llama_context_params params); - - LLAMA_API void llama_free_model(struct llama_model * model); - - LLAMA_API struct llama_context * llama_new_context_with_model( - struct llama_model * model, - struct llama_context_params params); - - - // Frees all allocated memory - LLAMA_API void llama_free(struct llama_context * ctx); - - // Returns 0 on success - LLAMA_API int llama_model_quantize( - const char * fname_inp, - const char * fname_out, - const llama_model_quantize_params * params); - - // Apply a LoRA adapter to a loaded model - // path_base_model is the path to a higher quality model to use as a base for - // the layers modified by the adapter. Can be NULL to use the current loaded model. - // The model needs to be reloaded before applying a new adapter, otherwise the adapter - // will be applied on top of the previous one - // Returns 0 on success - LLAMA_API DEPRECATED(int llama_apply_lora_from_file( - struct llama_context * ctx, - const char * path_lora, - const char * path_base_model, - int n_threads), - "please use llama_model_apply_lora_from_file instead"); - - LLAMA_API int llama_model_apply_lora_from_file( - const struct llama_model * model, - const char * path_lora, - const char * path_base_model, - int n_threads); - - // Returns the number of tokens in the KV cache - LLAMA_API int llama_get_kv_cache_token_count(const struct llama_context * ctx); - - // Sets the current rng seed. - LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed); - - // Returns the maximum size in bytes of the state (rng, logits, embedding - // and kv_cache) - will often be smaller after compacting tokens - LLAMA_API size_t llama_get_state_size(const struct llama_context * ctx); - - // Copies the state to the specified destination address. - // Destination needs to have allocated enough memory. - // Returns the number of bytes copied - LLAMA_API size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst); - - // Set the state reading from the specified address - // Returns the number of bytes read - LLAMA_API size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src); - - // Save/load session file - LLAMA_API bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out); - LLAMA_API bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count); - - // Run the llama inference to obtain the logits and probabilities for the next token. - // tokens + n_tokens is the provided batch of new tokens to process - // n_past is the number of tokens to use from previous eval calls - // Returns 0 on success - LLAMA_API int llama_eval( - struct llama_context * ctx, - const llama_token * tokens, - int n_tokens, - int n_past, - int n_threads); - - // Same as llama_eval, but use float matrix input directly. - LLAMA_API int llama_eval_embd( - struct llama_context * ctx, - const float * embd, - int n_tokens, - int n_past, - int n_threads); - - // Export a static computation graph for context of 511 and batch size of 1 - // NOTE: since this functionality is mostly for debugging and demonstration purposes, we hardcode these - // parameters here to keep things simple - // IMPORTANT: do not use for anything else other than debugging and testing! - LLAMA_API int llama_eval_export(struct llama_context * ctx, const char * fname); - - // Convert the provided text into tokens. - // The tokens pointer must be large enough to hold the resulting tokens. - // Returns the number of tokens on success, no more than n_max_tokens - // Returns a negative number on failure - the number of tokens that would have been returned - // TODO: not sure if correct - LLAMA_API int llama_tokenize( - struct llama_context * ctx, - const char * text, - llama_token * tokens, - int n_max_tokens, - bool add_bos); - - LLAMA_API int llama_tokenize_with_model( - const struct llama_model * model, - const char * text, - llama_token * tokens, - int n_max_tokens, - bool add_bos); - - LLAMA_API int llama_n_vocab(const struct llama_context * ctx); - LLAMA_API int llama_n_ctx (const struct llama_context * ctx); - LLAMA_API int llama_n_embd (const struct llama_context * ctx); - - LLAMA_API int llama_n_vocab_from_model(const struct llama_model * model); - LLAMA_API int llama_n_ctx_from_model (const struct llama_model * model); - LLAMA_API int llama_n_embd_from_model (const struct llama_model * model); - - // Get the vocabulary as output parameters. - // Returns number of results. - LLAMA_API int llama_get_vocab( - const struct llama_context * ctx, - const char * * strings, - float * scores, - int capacity); - - LLAMA_API int llama_get_vocab_from_model( - const struct llama_model * model, - const char * * strings, - float * scores, - int capacity); - - // Token logits obtained from the last call to llama_eval() - // The logits for the last token are stored in the last row - // Can be mutated in order to change the probabilities of the next token - // Rows: n_tokens - // Cols: n_vocab - LLAMA_API float * llama_get_logits(struct llama_context * ctx); - - // Get the embeddings for the input - // shape: [n_embd] (1-dimensional) - LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); - - // Token Id -> String. Uses the vocabulary in the provided context - LLAMA_API const char * llama_token_to_str( - const struct llama_context * ctx, - llama_token token); - - LLAMA_API const char * llama_token_to_str_with_model( - const struct llama_model * model, - llama_token token); - - // Special tokens - LLAMA_API llama_token llama_token_bos(); // beginning-of-sentence - LLAMA_API llama_token llama_token_eos(); // end-of-sentence - LLAMA_API llama_token llama_token_nl(); // next-line - - // Grammar - // - LLAMA_API struct llama_grammar * llama_grammar_init( - const llama_grammar_element ** rules, - size_t n_rules, - size_t start_rule_index); - - LLAMA_API void llama_grammar_free(struct llama_grammar * grammar); - - // Sampling functions - - /// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. - LLAMA_API void llama_sample_repetition_penalty(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float penalty); - - /// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. - LLAMA_API void llama_sample_frequency_and_presence_penalties(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float alpha_frequency, float alpha_presence); - - /// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806 - /// @param candidates A vector of `llama_token_data` containing the candidate tokens, the logits must be directly extracted from the original generation context without being sorted. - /// @params guidance_ctx A separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context. - /// @params scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance. - LLAMA_API void llama_sample_classifier_free_guidance( - struct llama_context * ctx, - llama_token_data_array * candidates, - struct llama_context * guidance_ctx, - float scale); - - /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. - LLAMA_API void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates); - - /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 - LLAMA_API void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep); - - /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 - LLAMA_API void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep); - - /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. - LLAMA_API void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep); - - /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. - LLAMA_API void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep); - LLAMA_API void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates, float temp); - - /// @details Apply constraints from grammar - LLAMA_API void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * candidates, const struct llama_grammar * grammar); - - /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. - /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. - /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. - /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. - /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. - LLAMA_API llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int m, float * mu); - - /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. - /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. - /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. - /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. - LLAMA_API llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu); - - /// @details Selects the token with the highest probability. - LLAMA_API llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates); - - /// @details Randomly selects a token from the candidates based on their probabilities. - LLAMA_API llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates); - - /// @details Accepts the sampled token into the grammar - LLAMA_API void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar * grammar, llama_token token); - - // Performance information - LLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx); - LLAMA_API void llama_print_timings(struct llama_context * ctx); - LLAMA_API void llama_reset_timings(struct llama_context * ctx); - - // Print system information - LLAMA_API const char * llama_print_system_info(void); - -#ifdef __cplusplus -} -#endif - -// Internal API to be implemented by llama.cpp and used by tests/benchmarks only -#ifdef LLAMA_API_INTERNAL - -#include -#include -struct ggml_tensor; - -const std::vector>& llama_internal_get_tensor_map(struct llama_context * ctx); - -#endif - -#endif // LLAMA_H +#ifndef LLAMA_H +#define LLAMA_H + +#include "ggml.h" +#ifdef GGML_USE_CUBLAS +#include "ggml-cuda.h" +#define LLAMA_MAX_DEVICES GGML_CUDA_MAX_DEVICES +#else +#define LLAMA_MAX_DEVICES 1 +#endif // GGML_USE_CUBLAS +#include +#include +#include + +#ifdef LLAMA_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef LLAMA_BUILD +# define LLAMA_API __declspec(dllexport) +# else +# define LLAMA_API __declspec(dllimport) +# endif +# else +# define LLAMA_API __attribute__ ((visibility ("default"))) +# endif +#else +# define LLAMA_API +#endif + +#ifdef __GNUC__ +# define DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define DEPRECATED(func, hint) func +#endif + +#define LLAMA_DEFAULT_SEED 0xFFFFFFFF + +#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL) +// Defined when llama.cpp is compiled with support for offloading model layers to GPU. +#define LLAMA_SUPPORTS_GPU_OFFLOAD +#endif + +#ifndef LLAMA_DEFAULT_RMS_EPS +#define LLAMA_DEFAULT_RMS_EPS 5e-6f +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + // + // C interface + // + // TODO: show sample usage + // + + struct llama_model; + struct llama_context; + + typedef int llama_token; + + typedef struct llama_token_data { + llama_token id; // token id + float logit; // log-odds of the token + float p; // probability of the token + } llama_token_data; + + typedef struct llama_token_data_array { + llama_token_data * data; + size_t size; + bool sorted; + } llama_token_data_array; + + typedef void (*llama_progress_callback)(float progress, void *ctx); + + struct llama_context_params { + uint32_t seed; // RNG seed, -1 for random + int32_t n_ctx; // text context + int32_t n_batch; // prompt processing batch size + int32_t n_gqa; // grouped-query attention (TEMP - will be moved to model hparams) + float rms_norm_eps; // rms norm epsilon (TEMP - will be moved to model hparams) + int32_t n_gpu_layers; // number of layers to store in VRAM + int32_t main_gpu; // the GPU that is used for scratch and small tensors + + const float * tensor_split; // how to split layers across multiple GPUs (size: LLAMA_MAX_DEVICES) + + // ref: https://github.com/ggerganov/llama.cpp/pull/2054 + float rope_freq_base; // RoPE base frequency + float rope_freq_scale; // RoPE frequency scaling factor + + // called with a progress value between 0 and 1, pass NULL to disable + llama_progress_callback progress_callback; + // context pointer passed to the progress callback + void * progress_callback_user_data; + + // Keep the booleans together to avoid misalignment during copy-by-value. + bool low_vram; // if true, reduce VRAM usage at the cost of performance + bool f16_kv; // use fp16 for KV cache + bool logits_all; // the llama_eval() call computes all logits, not just the last one + bool vocab_only; // only load the vocabulary, no weights + bool use_mmap; // use mmap if possible + bool use_mlock; // force system to keep model in RAM + bool embedding; // embedding mode only + }; + // model file types + enum llama_ftype { + LLAMA_FTYPE_ALL_F32 = 0, + LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed + // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed + LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K = 10,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_S = 11,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_M = 12,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_L = 13,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_S = 14,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_M = 15,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_S = 16,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_M = 17,// except 1d tensors + LLAMA_FTYPE_MOSTLY_Q6_K = 18,// except 1d tensors + }; + + // model quantization parameters + typedef struct llama_model_quantize_params { + int nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + } llama_model_quantize_params; + + // grammar types + struct llama_grammar; + + // grammar element type + enum llama_gretype { + // end of rule definition + LLAMA_GRETYPE_END = 0, + + // start of alternate definition for rule + LLAMA_GRETYPE_ALT = 1, + + // non-terminal element: reference to rule + LLAMA_GRETYPE_RULE_REF = 2, + + // terminal element: character (code point) + LLAMA_GRETYPE_CHAR = 3, + + // inverse char(s) ([^a], [^a-b] [^abc]) + LLAMA_GRETYPE_CHAR_NOT = 4, + + // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to + // be an inclusive range ([a-z]) + LLAMA_GRETYPE_CHAR_RNG_UPPER = 5, + + // modifies a preceding LLAMA_GRETYPE_CHAR or + // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) + LLAMA_GRETYPE_CHAR_ALT = 6, + }; + + typedef struct llama_grammar_element { + enum llama_gretype type; + uint32_t value; // Unicode code point or rule ID + } llama_grammar_element; + + // performance timing information + struct llama_timings { + double t_start_ms; + double t_end_ms; + double t_load_ms; + double t_sample_ms; + double t_p_eval_ms; + double t_eval_ms; + + int32_t n_sample; + int32_t n_p_eval; + int32_t n_eval; + }; + + LLAMA_API int llama_max_devices(); + + LLAMA_API struct llama_context_params llama_context_default_params(); + LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(); + + LLAMA_API bool llama_mmap_supported(); + LLAMA_API bool llama_mlock_supported(); + + // TODO: not great API - very likely to change + // Initialize the llama + ggml backend + // If numa is true, use NUMA optimizations + // Call once at the start of the program + LLAMA_API void llama_backend_init(bool numa); + // Call once at the end of the program - currently only used for MPI + LLAMA_API void llama_backend_free(); + + LLAMA_API int64_t llama_time_us(); + + LLAMA_API struct llama_model * llama_load_model_from_file( + const char * path_model, + struct llama_context_params params); + + LLAMA_API void llama_free_model(struct llama_model * model); + + LLAMA_API struct llama_context * llama_new_context_with_model( + struct llama_model * model, + struct llama_context_params params); + + + // Frees all allocated memory + LLAMA_API void llama_free(struct llama_context * ctx); + + // Returns 0 on success + LLAMA_API int llama_model_quantize( + const char * fname_inp, + const char * fname_out, + const llama_model_quantize_params * params); + + // Apply a LoRA adapter to a loaded model + // path_base_model is the path to a higher quality model to use as a base for + // the layers modified by the adapter. Can be NULL to use the current loaded model. + // The model needs to be reloaded before applying a new adapter, otherwise the adapter + // will be applied on top of the previous one + // Returns 0 on success + LLAMA_API DEPRECATED(int llama_apply_lora_from_file( + struct llama_context * ctx, + const char * path_lora, + const char * path_base_model, + int n_threads), + "please use llama_model_apply_lora_from_file instead"); + + LLAMA_API int llama_model_apply_lora_from_file( + const struct llama_model * model, + const char * path_lora, + const char * path_base_model, + int n_threads); + + // Returns the number of tokens in the KV cache + LLAMA_API int llama_get_kv_cache_token_count(const struct llama_context * ctx); + + // Sets the current rng seed. + LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed); + + // Returns the maximum size in bytes of the state (rng, logits, embedding + // and kv_cache) - will often be smaller after compacting tokens + LLAMA_API size_t llama_get_state_size(const struct llama_context * ctx); + + // Copies the state to the specified destination address. + // Destination needs to have allocated enough memory. + // Returns the number of bytes copied + LLAMA_API size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst); + + // Set the state reading from the specified address + // Returns the number of bytes read + LLAMA_API size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src); + + // Save/load session file + LLAMA_API bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out); + LLAMA_API bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count); + + // Run the llama inference to obtain the logits and probabilities for the next token. + // tokens + n_tokens is the provided batch of new tokens to process + // n_past is the number of tokens to use from previous eval calls + // Returns 0 on success + LLAMA_API int llama_eval( + struct llama_context * ctx, + const llama_token * tokens, + int n_tokens, + int n_past, + int n_threads); + + // Same as llama_eval, but use float matrix input directly. + LLAMA_API int llama_eval_embd( + struct llama_context * ctx, + const float * embd, + int n_tokens, + int n_past, + int n_threads); + + // Export a static computation graph for context of 511 and batch size of 1 + // NOTE: since this functionality is mostly for debugging and demonstration purposes, we hardcode these + // parameters here to keep things simple + // IMPORTANT: do not use for anything else other than debugging and testing! + LLAMA_API int llama_eval_export(struct llama_context * ctx, const char * fname); + + // Convert the provided text into tokens. + // The tokens pointer must be large enough to hold the resulting tokens. + // Returns the number of tokens on success, no more than n_max_tokens + // Returns a negative number on failure - the number of tokens that would have been returned + // TODO: not sure if correct + LLAMA_API int llama_tokenize( + struct llama_context * ctx, + const char * text, + llama_token * tokens, + int n_max_tokens, + bool add_bos); + + LLAMA_API int llama_tokenize_with_model( + const struct llama_model * model, + const char * text, + llama_token * tokens, + int n_max_tokens, + bool add_bos); + + LLAMA_API int llama_n_vocab(const struct llama_context * ctx); + LLAMA_API int llama_n_ctx (const struct llama_context * ctx); + LLAMA_API int llama_n_embd (const struct llama_context * ctx); + + LLAMA_API int llama_n_vocab_from_model(const struct llama_model * model); + LLAMA_API int llama_n_ctx_from_model (const struct llama_model * model); + LLAMA_API int llama_n_embd_from_model (const struct llama_model * model); + + // Get the vocabulary as output parameters. + // Returns number of results. + LLAMA_API int llama_get_vocab( + const struct llama_context * ctx, + const char * * strings, + float * scores, + int capacity); + + LLAMA_API int llama_get_vocab_from_model( + const struct llama_model * model, + const char * * strings, + float * scores, + int capacity); + + // Token logits obtained from the last call to llama_eval() + // The logits for the last token are stored in the last row + // Can be mutated in order to change the probabilities of the next token + // Rows: n_tokens + // Cols: n_vocab + LLAMA_API float * llama_get_logits(struct llama_context * ctx); + + // Get the embeddings for the input + // shape: [n_embd] (1-dimensional) + LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); + + // Token Id -> String. Uses the vocabulary in the provided context + LLAMA_API const char * llama_token_to_str( + const struct llama_context * ctx, + llama_token token); + + LLAMA_API const char * llama_token_to_str_with_model( + const struct llama_model * model, + llama_token token); + + // Special tokens + LLAMA_API llama_token llama_token_bos(); // beginning-of-sentence + LLAMA_API llama_token llama_token_eos(); // end-of-sentence + LLAMA_API llama_token llama_token_nl(); // next-line + + // Grammar + // + LLAMA_API struct llama_grammar * llama_grammar_init( + const llama_grammar_element ** rules, + size_t n_rules, + size_t start_rule_index); + + LLAMA_API void llama_grammar_free(struct llama_grammar * grammar); + + // Sampling functions + + /// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. + LLAMA_API void llama_sample_repetition_penalty(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float penalty); + + /// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. + LLAMA_API void llama_sample_frequency_and_presence_penalties(struct llama_context * ctx, llama_token_data_array * candidates, const llama_token * last_tokens, size_t last_tokens_size, float alpha_frequency, float alpha_presence); + + /// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806 + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, the logits must be directly extracted from the original generation context without being sorted. + /// @params guidance_ctx A separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context. + /// @params scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance. + LLAMA_API void llama_sample_classifier_free_guidance( + struct llama_context * ctx, + llama_token_data_array * candidates, + struct llama_context * guidance_ctx, + float scale); + + /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. + LLAMA_API void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * candidates); + + /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + LLAMA_API void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep); + + /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + LLAMA_API void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep); + + /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. + LLAMA_API void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep); + + /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. + LLAMA_API void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep); + LLAMA_API void llama_sample_temperature(struct llama_context * ctx, llama_token_data_array * candidates, float temp); + + /// @details Apply constraints from grammar + LLAMA_API void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * candidates, const struct llama_grammar * grammar); + + /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int m, float * mu); + + /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API llama_token llama_sample_token_mirostat_v2(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, float * mu); + + /// @details Selects the token with the highest probability. + LLAMA_API llama_token llama_sample_token_greedy(struct llama_context * ctx, llama_token_data_array * candidates); + + /// @details Randomly selects a token from the candidates based on their probabilities. + LLAMA_API llama_token llama_sample_token(struct llama_context * ctx, llama_token_data_array * candidates); + + /// @details Accepts the sampled token into the grammar + LLAMA_API void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar * grammar, llama_token token); + + // Performance information + LLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx); + LLAMA_API void llama_print_timings(struct llama_context * ctx); + LLAMA_API void llama_reset_timings(struct llama_context * ctx); + + // Print system information + LLAMA_API const char * llama_print_system_info(void); + +#ifdef __cplusplus +} +#endif + +// Internal API to be implemented by llama.cpp and used by tests/benchmarks only +#ifdef LLAMA_API_INTERNAL + +#include +#include +struct ggml_tensor; + +const std::vector>& llama_internal_get_tensor_map(struct llama_context * ctx); + +#endif + +#endif // LLAMA_H diff --git a/gguf-util.h b/gguf-util.h index 6bbabf667911e..774ae57eef3db 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -1,567 +1,567 @@ -// GGUF counterpart of llama-util.h. -// we may consider making it a part of ggml.c once GGUF work is complete. -// this will require extra work to migrate this to pure C. -// Contains wrappers around OS interfaces. - -#ifndef GGUF_UTIL_H -#define GGUF_UTIL_H - -#include "ggml.h" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#ifdef __has_include - #if __has_include() - #include - #if defined(_POSIX_MAPPED_FILES) - #include - #endif - #if defined(_POSIX_MEMLOCK_RANGE) - #include - #endif - #endif -#endif - -#if defined(_WIN32) - #define WIN32_LEAN_AND_MEAN - #ifndef NOMINMAX - #define NOMINMAX - #endif - #include - #include - #include // for _fseeki64 -#endif - -#ifdef __GNUC__ -#ifdef __MINGW32__ -__attribute__((format(gnu_printf, 1, 2))) -#else -__attribute__((format(printf, 1, 2))) -#endif -#endif -static std::string format(const char * fmt, ...) { - va_list ap, ap2; - va_start(ap, fmt); - va_copy(ap2, ap); - int size = vsnprintf(NULL, 0, fmt, ap); - GGML_ASSERT(size >= 0 && size < INT_MAX); - std::vector buf(size + 1); - int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2); - GGML_ASSERT(size2 == size); - va_end(ap2); - va_end(ap); - return std::string(buf.data(), size); -} - -template -static std::string to_string(const T & val) { - std::stringstream ss; - ss << val; - return ss.str(); -} - -// TODO: can we merge this one and gguf_context? -struct gguf_file { - // use FILE * so we don't have to re-open the file to mmap - FILE * fp; - size_t size; - - gguf_file(const char * fname, const char * mode) { - fp = std::fopen(fname, mode); - if (fp == NULL) { - throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); - } - seek(0, SEEK_END); - size = tell(); - seek(0, SEEK_SET); - } - - size_t tell() const { -#ifdef _WIN32 - __int64 ret = _ftelli64(fp); -#else - long ret = std::ftell(fp); -#endif - GGML_ASSERT(ret != -1); // this really shouldn't fail - return (size_t) ret; - } - - void seek(size_t offset, int whence) { -#ifdef _WIN32 - int ret = _fseeki64(fp, (__int64) offset, whence); -#else - int ret = std::fseek(fp, (long) offset, whence); -#endif - GGML_ASSERT(ret == 0); // same - } - - size_t write_str(const std::string & val) { - size_t total_written = 0; - const int32_t n = val.size(); - fwrite((const char *) &n, sizeof(n), 1, fp); - total_written += sizeof(n); - fwrite(val.c_str(), n, 1, fp); - total_written += n; - - return total_written; - } - - size_t write_i32(int32_t val) { - fwrite((const char *) &val, sizeof(val), 1, fp); - return sizeof(val); - } - - size_t write_u64(size_t val) { - fwrite((const char *) &val, sizeof(val), 1, fp); - return sizeof(val); - } - - template - void write_val(const std::string & key, enum gguf_type type, const T & val) { - write_str(key); - fwrite((const char *) &type, sizeof(type), 1, fp); - fwrite((const char *) &val, sizeof(val), 1, fp); - } - - template - void write_arr(const std::string & key, enum gguf_type type, const std::vector & val) { - write_str(key); - { - const enum gguf_type tarr = GGUF_TYPE_ARRAY; - fwrite((const char *) &tarr, sizeof(tarr), 1, fp); - } - - const int32_t n = val.size(); - fwrite((const char *) &type, sizeof(type), 1, fp); - fwrite((const char *) &n, sizeof(n), 1, fp); - fwrite(val.data(), sizeof(T), n, fp); - } - - void write_str(const std::string & key, enum gguf_type type, const std::string & val) { - write_str(key); - fwrite((const char *) &type, sizeof(type), 1, fp); - - const int32_t n = val.size(); - fwrite((const char *) &n, sizeof(n), 1, fp); - fwrite(val.c_str(), n, 1, fp); - } - - void write_str(const std::string & key, enum gguf_type type, const std::vector & val) { - write_str(key); - { - const enum gguf_type tarr = GGUF_TYPE_ARRAY; - fwrite((const char *) &tarr, sizeof(tarr), 1, fp); - } - - const int32_t n = val.size(); - fwrite((const char *) &type, sizeof(type), 1, fp); - fwrite((const char *) &n, sizeof(n), 1, fp); - for (int i = 0; i < n; ++i) { - const int32_t nstr = val[i].size(); - fwrite((const char *) &nstr, sizeof(nstr), 1, fp); - fwrite(val[i].c_str(), nstr, 1, fp); - } - } - - void write_zeros(size_t count) { - for (size_t i = 0; i < count; ++i) { - fputc(0, fp); - } - } - - void read_raw(void * ptr, size_t len) const { - if (len == 0) { - return; - } - errno = 0; - std::size_t ret = std::fread(ptr, len, 1, fp); - if (ferror(fp)) { - throw std::runtime_error(format("read error: %s", strerror(errno))); - } - if (ret != 1) { - throw std::runtime_error(std::string("unexpectedly reached end of file")); - } - } - - void write_raw(const void * ptr, size_t len) const { - if (len == 0) { - return; - } - errno = 0; - size_t ret = std::fwrite(ptr, len, 1, fp); - if (ret != 1) { - throw std::runtime_error(format("write error: %s", strerror(errno))); - } - } - - ~gguf_file() { - if (fp) { - std::fclose(fp); - } - } -}; - -#if defined(_WIN32) -static std::string gguf_format_win_err(DWORD err) { - LPSTR buf; - size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL); - if (!size) { - return "FormatMessageA failed"; - } - std::string ret(buf, size); - LocalFree(buf); - return ret; -} -#endif - -struct gguf_mmap { - void * addr; - size_t size; - - gguf_mmap(const gguf_mmap &) = delete; - -#ifdef _POSIX_MAPPED_FILES - static constexpr bool SUPPORTED = true; - - gguf_mmap(struct gguf_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) { - size = file->size; - int fd = fileno(file->fp); - int flags = MAP_SHARED; - // prefetch/readahead impairs performance on NUMA systems - if (numa) { prefetch = 0; } -#ifdef __linux__ - if (prefetch) { flags |= MAP_POPULATE; } -#endif - addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0); - if (addr == MAP_FAILED) { - throw std::runtime_error(format("mmap failed: %s", strerror(errno))); - } - - if (prefetch > 0) { - // Advise the kernel to preload the mapped memory - if (madvise(addr, std::min(file->size, prefetch), MADV_WILLNEED)) { - fprintf(stderr, "warning: madvise(.., MADV_WILLNEED) failed: %s\n", - strerror(errno)); - } - } - if (numa) { - // advise the kernel not to use readahead - // (because the next page might not belong on the same node) - if (madvise(addr, file->size, MADV_RANDOM)) { - fprintf(stderr, "warning: madvise(.., MADV_RANDOM) failed: %s\n", - strerror(errno)); - } - } - } - - ~gguf_mmap() { - munmap(addr, size); - } -#elif defined(_WIN32) - static constexpr bool SUPPORTED = true; - - gguf_mmap(struct llama_file * file, bool prefetch = true, bool numa = false) { - (void) numa; - - size = file->size; - - HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp)); - - HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL); - DWORD error = GetLastError(); - - if (hMapping == NULL) { - throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str())); - } - - addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); - error = GetLastError(); - CloseHandle(hMapping); - - if (addr == NULL) { - throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str())); - } - - #if _WIN32_WINNT >= _WIN32_WINNT_WIN8 - if (prefetch) { - // Advise the kernel to preload the mapped memory - WIN32_MEMORY_RANGE_ENTRY range; - range.VirtualAddress = addr; - range.NumberOfBytes = (SIZE_T)size; - if (!PrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) { - fprintf(stderr, "warning: PrefetchVirtualMemory failed: %s\n", - gguf_format_win_err(GetLastError()).c_str()); - } - } - #else - #pragma message("warning: You are building for pre-Windows 8; prefetch not supported") - #endif // _WIN32_WINNT >= _WIN32_WINNT_WIN8 - } - - ~gguf_mmap() { - if (!UnmapViewOfFile(addr)) { - fprintf(stderr, "warning: UnmapViewOfFile failed: %s\n", - llama_format_win_err(GetLastError()).c_str()); - } - } -#else - static constexpr bool SUPPORTED = false; - - gguf_mmap(struct llama_file *, bool prefetch = true, bool numa = false) { - (void) prefetch; - (void) numa; - - throw std::runtime_error(std::string("mmap not supported")); - } -#endif -}; - -// Represents some region of memory being locked using mlock or VirtualLock; -// will automatically unlock on destruction. -struct gguf_mlock { - void * addr = NULL; - size_t size = 0; - bool failed_already = false; - - gguf_mlock() {} - gguf_mlock(const gguf_mlock &) = delete; - - ~gguf_mlock() { - if (size) { - raw_unlock(addr, size); - } - } - - void init(void * ptr) { - GGML_ASSERT(addr == NULL && size == 0); - addr = ptr; - } - - void grow_to(size_t target_size) { - GGML_ASSERT(addr); - if (failed_already) { - return; - } - size_t granularity = lock_granularity(); - target_size = (target_size + granularity - 1) & ~(granularity - 1); - if (target_size > size) { - if (raw_lock((uint8_t *) addr + size, target_size - size)) { - size = target_size; - } else { - failed_already = true; - } - } - } - -#ifdef _POSIX_MEMLOCK_RANGE - static constexpr bool SUPPORTED = true; - - size_t lock_granularity() { - return (size_t) sysconf(_SC_PAGESIZE); - } - - #ifdef __APPLE__ - #define MLOCK_SUGGESTION \ - "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \ - "decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MLOCK (ulimit -l).\n" - #else - #define MLOCK_SUGGESTION \ - "Try increasing RLIMIT_MLOCK ('ulimit -l' as root).\n" - #endif - - bool raw_lock(const void * addr, size_t size) { - if (!mlock(addr, size)) { - return true; - } else { - char* errmsg = std::strerror(errno); - bool suggest = (errno == ENOMEM); - - // Check if the resource limit is fine after all - struct rlimit lock_limit; - if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) - suggest = false; - if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size)) - suggest = false; - - fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s", - size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : ""); - return false; - } - } - - #undef MLOCK_SUGGESTION - - void raw_unlock(void * addr, size_t size) { - if (munlock(addr, size)) { - fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno)); - } - } -#elif defined(_WIN32) - static constexpr bool SUPPORTED = true; - - size_t lock_granularity() { - SYSTEM_INFO si; - GetSystemInfo(&si); - return (size_t) si.dwPageSize; - } - - bool raw_lock(void * ptr, size_t len) { - for (int tries = 1; ; tries++) { - if (VirtualLock(ptr, len)) { - return true; - } - if (tries == 2) { - fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n", - len, size, llama_format_win_err(GetLastError()).c_str()); - return false; - } - - // It failed but this was only the first try; increase the working - // set size and try again. - SIZE_T min_ws_size, max_ws_size; - if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) { - fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n", - gguf_format_win_err(GetLastError()).c_str()); - return false; - } - // Per MSDN: "The maximum number of pages that a process can lock - // is equal to the number of pages in its minimum working set minus - // a small overhead." - // Hopefully a megabyte is enough overhead: - size_t increment = len + 1048576; - // The minimum must be <= the maximum, so we need to increase both: - min_ws_size += increment; - max_ws_size += increment; - if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) { - fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n", - gguf_format_win_err(GetLastError()).c_str()); - return false; - } - } - } - - void raw_unlock(void * ptr, size_t len) { - if (!VirtualUnlock(ptr, len)) { - fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n", - gguf_format_win_err(GetLastError()).c_str()); - } - } -#else - static constexpr bool SUPPORTED = false; - - size_t lock_granularity() { - return (size_t) 65536; - } - - bool raw_lock(const void * addr, size_t len) { - fprintf(stderr, "warning: mlock not supported on this system\n"); - return false; - } - - void raw_unlock(const void * addr, size_t len) {} -#endif -}; - -// Replacement for std::vector that doesn't require zero-initialization. -struct gguf_buffer { - uint8_t * addr = NULL; - size_t size = 0; - - gguf_buffer() = default; - - void resize(size_t len) { -#ifdef GGML_USE_METAL - free(addr); - int result = posix_memalign((void **) &addr, getpagesize(), len); - if (result == 0) { - memset(addr, 0, len); - } - else { - addr = NULL; - } -#else - delete[] addr; - addr = new uint8_t[len]; -#endif - size = len; - } - - ~gguf_buffer() { -#ifdef GGML_USE_METAL - free(addr); -#else - delete[] addr; -#endif - addr = NULL; - } - - // disable copy and move - gguf_buffer(const gguf_buffer&) = delete; - gguf_buffer(gguf_buffer&&) = delete; - gguf_buffer& operator=(const gguf_buffer&) = delete; - gguf_buffer& operator=(gguf_buffer&&) = delete; -}; - -#ifdef GGML_USE_CUBLAS -#include "ggml-cuda.h" -struct gguf_ctx_buffer { - uint8_t * addr = NULL; - bool is_cuda; - size_t size = 0; - - gguf_ctx_buffer() = default; - - void resize(size_t size) { - free(); - - addr = (uint8_t *) ggml_cuda_host_malloc(size); - if (addr) { - is_cuda = true; - } - else { - // fall back to pageable memory - addr = new uint8_t[size]; - is_cuda = false; - } - this->size = size; - } - - void free() { - if (addr) { - if (is_cuda) { - ggml_cuda_host_free(addr); - } - else { - delete[] addr; - } - } - addr = NULL; - } - - ~gguf_ctx_buffer() { - free(); - } - - // disable copy and move - gguf_ctx_buffer(const gguf_ctx_buffer&) = delete; - gguf_ctx_buffer(gguf_ctx_buffer&&) = delete; - gguf_ctx_buffer& operator=(const gguf_ctx_buffer&) = delete; - gguf_ctx_buffer& operator=(gguf_ctx_buffer&&) = delete; -}; -#else -typedef gguf_buffer gguf_ctx_buffer; -#endif - -#endif +// GGUF counterpart of llama-util.h. +// we may consider making it a part of ggml.c once GGUF work is complete. +// this will require extra work to migrate this to pure C. +// Contains wrappers around OS interfaces. + +#ifndef GGUF_UTIL_H +#define GGUF_UTIL_H + +#include "ggml.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef __has_include + #if __has_include() + #include + #if defined(_POSIX_MAPPED_FILES) + #include + #endif + #if defined(_POSIX_MEMLOCK_RANGE) + #include + #endif + #endif +#endif + +#if defined(_WIN32) + #define WIN32_LEAN_AND_MEAN + #ifndef NOMINMAX + #define NOMINMAX + #endif + #include + #include + #include // for _fseeki64 +#endif + +#ifdef __GNUC__ +#ifdef __MINGW32__ +__attribute__((format(gnu_printf, 1, 2))) +#else +__attribute__((format(printf, 1, 2))) +#endif +#endif +static std::string format(const char * fmt, ...) { + va_list ap, ap2; + va_start(ap, fmt); + va_copy(ap2, ap); + int size = vsnprintf(NULL, 0, fmt, ap); + GGML_ASSERT(size >= 0 && size < INT_MAX); + std::vector buf(size + 1); + int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2); + GGML_ASSERT(size2 == size); + va_end(ap2); + va_end(ap); + return std::string(buf.data(), size); +} + +template +static std::string to_string(const T & val) { + std::stringstream ss; + ss << val; + return ss.str(); +} + +// TODO: can we merge this one and gguf_context? +struct gguf_file { + // use FILE * so we don't have to re-open the file to mmap + FILE * fp; + size_t size; + + gguf_file(const char * fname, const char * mode) { + fp = std::fopen(fname, mode); + if (fp == NULL) { + throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); + } + seek(0, SEEK_END); + size = tell(); + seek(0, SEEK_SET); + } + + size_t tell() const { +#ifdef _WIN32 + __int64 ret = _ftelli64(fp); +#else + long ret = std::ftell(fp); +#endif + GGML_ASSERT(ret != -1); // this really shouldn't fail + return (size_t) ret; + } + + void seek(size_t offset, int whence) { +#ifdef _WIN32 + int ret = _fseeki64(fp, (__int64) offset, whence); +#else + int ret = std::fseek(fp, (long) offset, whence); +#endif + GGML_ASSERT(ret == 0); // same + } + + size_t write_str(const std::string & val) { + size_t total_written = 0; + const int32_t n = val.size(); + fwrite((const char *) &n, sizeof(n), 1, fp); + total_written += sizeof(n); + fwrite(val.c_str(), n, 1, fp); + total_written += n; + + return total_written; + } + + size_t write_i32(int32_t val) { + fwrite((const char *) &val, sizeof(val), 1, fp); + return sizeof(val); + } + + size_t write_u64(size_t val) { + fwrite((const char *) &val, sizeof(val), 1, fp); + return sizeof(val); + } + + template + void write_val(const std::string & key, enum gguf_type type, const T & val) { + write_str(key); + fwrite((const char *) &type, sizeof(type), 1, fp); + fwrite((const char *) &val, sizeof(val), 1, fp); + } + + template + void write_arr(const std::string & key, enum gguf_type type, const std::vector & val) { + write_str(key); + { + const enum gguf_type tarr = GGUF_TYPE_ARRAY; + fwrite((const char *) &tarr, sizeof(tarr), 1, fp); + } + + const int32_t n = val.size(); + fwrite((const char *) &type, sizeof(type), 1, fp); + fwrite((const char *) &n, sizeof(n), 1, fp); + fwrite(val.data(), sizeof(T), n, fp); + } + + void write_str(const std::string & key, enum gguf_type type, const std::string & val) { + write_str(key); + fwrite((const char *) &type, sizeof(type), 1, fp); + + const int32_t n = val.size(); + fwrite((const char *) &n, sizeof(n), 1, fp); + fwrite(val.c_str(), n, 1, fp); + } + + void write_str(const std::string & key, enum gguf_type type, const std::vector & val) { + write_str(key); + { + const enum gguf_type tarr = GGUF_TYPE_ARRAY; + fwrite((const char *) &tarr, sizeof(tarr), 1, fp); + } + + const int32_t n = val.size(); + fwrite((const char *) &type, sizeof(type), 1, fp); + fwrite((const char *) &n, sizeof(n), 1, fp); + for (int i = 0; i < n; ++i) { + const int32_t nstr = val[i].size(); + fwrite((const char *) &nstr, sizeof(nstr), 1, fp); + fwrite(val[i].c_str(), nstr, 1, fp); + } + } + + void write_zeros(size_t count) { + for (size_t i = 0; i < count; ++i) { + fputc(0, fp); + } + } + + void read_raw(void * ptr, size_t len) const { + if (len == 0) { + return; + } + errno = 0; + std::size_t ret = std::fread(ptr, len, 1, fp); + if (ferror(fp)) { + throw std::runtime_error(format("read error: %s", strerror(errno))); + } + if (ret != 1) { + throw std::runtime_error(std::string("unexpectedly reached end of file")); + } + } + + void write_raw(const void * ptr, size_t len) const { + if (len == 0) { + return; + } + errno = 0; + size_t ret = std::fwrite(ptr, len, 1, fp); + if (ret != 1) { + throw std::runtime_error(format("write error: %s", strerror(errno))); + } + } + + ~gguf_file() { + if (fp) { + std::fclose(fp); + } + } +}; + +#if defined(_WIN32) +static std::string gguf_format_win_err(DWORD err) { + LPSTR buf; + size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL); + if (!size) { + return "FormatMessageA failed"; + } + std::string ret(buf, size); + LocalFree(buf); + return ret; +} +#endif + +struct gguf_mmap { + void * addr; + size_t size; + + gguf_mmap(const gguf_mmap &) = delete; + +#ifdef _POSIX_MAPPED_FILES + static constexpr bool SUPPORTED = true; + + gguf_mmap(struct gguf_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) { + size = file->size; + int fd = fileno(file->fp); + int flags = MAP_SHARED; + // prefetch/readahead impairs performance on NUMA systems + if (numa) { prefetch = 0; } +#ifdef __linux__ + if (prefetch) { flags |= MAP_POPULATE; } +#endif + addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0); + if (addr == MAP_FAILED) { + throw std::runtime_error(format("mmap failed: %s", strerror(errno))); + } + + if (prefetch > 0) { + // Advise the kernel to preload the mapped memory + if (madvise(addr, std::min(file->size, prefetch), MADV_WILLNEED)) { + fprintf(stderr, "warning: madvise(.., MADV_WILLNEED) failed: %s\n", + strerror(errno)); + } + } + if (numa) { + // advise the kernel not to use readahead + // (because the next page might not belong on the same node) + if (madvise(addr, file->size, MADV_RANDOM)) { + fprintf(stderr, "warning: madvise(.., MADV_RANDOM) failed: %s\n", + strerror(errno)); + } + } + } + + ~gguf_mmap() { + munmap(addr, size); + } +#elif defined(_WIN32) + static constexpr bool SUPPORTED = true; + + gguf_mmap(struct llama_file * file, bool prefetch = true, bool numa = false) { + (void) numa; + + size = file->size; + + HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp)); + + HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL); + DWORD error = GetLastError(); + + if (hMapping == NULL) { + throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str())); + } + + addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0); + error = GetLastError(); + CloseHandle(hMapping); + + if (addr == NULL) { + throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str())); + } + + #if _WIN32_WINNT >= _WIN32_WINNT_WIN8 + if (prefetch) { + // Advise the kernel to preload the mapped memory + WIN32_MEMORY_RANGE_ENTRY range; + range.VirtualAddress = addr; + range.NumberOfBytes = (SIZE_T)size; + if (!PrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) { + fprintf(stderr, "warning: PrefetchVirtualMemory failed: %s\n", + gguf_format_win_err(GetLastError()).c_str()); + } + } + #else + #pragma message("warning: You are building for pre-Windows 8; prefetch not supported") + #endif // _WIN32_WINNT >= _WIN32_WINNT_WIN8 + } + + ~gguf_mmap() { + if (!UnmapViewOfFile(addr)) { + fprintf(stderr, "warning: UnmapViewOfFile failed: %s\n", + llama_format_win_err(GetLastError()).c_str()); + } + } +#else + static constexpr bool SUPPORTED = false; + + gguf_mmap(struct llama_file *, bool prefetch = true, bool numa = false) { + (void) prefetch; + (void) numa; + + throw std::runtime_error(std::string("mmap not supported")); + } +#endif +}; + +// Represents some region of memory being locked using mlock or VirtualLock; +// will automatically unlock on destruction. +struct gguf_mlock { + void * addr = NULL; + size_t size = 0; + bool failed_already = false; + + gguf_mlock() {} + gguf_mlock(const gguf_mlock &) = delete; + + ~gguf_mlock() { + if (size) { + raw_unlock(addr, size); + } + } + + void init(void * ptr) { + GGML_ASSERT(addr == NULL && size == 0); + addr = ptr; + } + + void grow_to(size_t target_size) { + GGML_ASSERT(addr); + if (failed_already) { + return; + } + size_t granularity = lock_granularity(); + target_size = (target_size + granularity - 1) & ~(granularity - 1); + if (target_size > size) { + if (raw_lock((uint8_t *) addr + size, target_size - size)) { + size = target_size; + } else { + failed_already = true; + } + } + } + +#ifdef _POSIX_MEMLOCK_RANGE + static constexpr bool SUPPORTED = true; + + size_t lock_granularity() { + return (size_t) sysconf(_SC_PAGESIZE); + } + + #ifdef __APPLE__ + #define MLOCK_SUGGESTION \ + "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \ + "decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MLOCK (ulimit -l).\n" + #else + #define MLOCK_SUGGESTION \ + "Try increasing RLIMIT_MLOCK ('ulimit -l' as root).\n" + #endif + + bool raw_lock(const void * addr, size_t size) { + if (!mlock(addr, size)) { + return true; + } else { + char* errmsg = std::strerror(errno); + bool suggest = (errno == ENOMEM); + + // Check if the resource limit is fine after all + struct rlimit lock_limit; + if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) + suggest = false; + if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size)) + suggest = false; + + fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s", + size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : ""); + return false; + } + } + + #undef MLOCK_SUGGESTION + + void raw_unlock(void * addr, size_t size) { + if (munlock(addr, size)) { + fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno)); + } + } +#elif defined(_WIN32) + static constexpr bool SUPPORTED = true; + + size_t lock_granularity() { + SYSTEM_INFO si; + GetSystemInfo(&si); + return (size_t) si.dwPageSize; + } + + bool raw_lock(void * ptr, size_t len) { + for (int tries = 1; ; tries++) { + if (VirtualLock(ptr, len)) { + return true; + } + if (tries == 2) { + fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n", + len, size, llama_format_win_err(GetLastError()).c_str()); + return false; + } + + // It failed but this was only the first try; increase the working + // set size and try again. + SIZE_T min_ws_size, max_ws_size; + if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) { + fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n", + gguf_format_win_err(GetLastError()).c_str()); + return false; + } + // Per MSDN: "The maximum number of pages that a process can lock + // is equal to the number of pages in its minimum working set minus + // a small overhead." + // Hopefully a megabyte is enough overhead: + size_t increment = len + 1048576; + // The minimum must be <= the maximum, so we need to increase both: + min_ws_size += increment; + max_ws_size += increment; + if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) { + fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n", + gguf_format_win_err(GetLastError()).c_str()); + return false; + } + } + } + + void raw_unlock(void * ptr, size_t len) { + if (!VirtualUnlock(ptr, len)) { + fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n", + gguf_format_win_err(GetLastError()).c_str()); + } + } +#else + static constexpr bool SUPPORTED = false; + + size_t lock_granularity() { + return (size_t) 65536; + } + + bool raw_lock(const void * addr, size_t len) { + fprintf(stderr, "warning: mlock not supported on this system\n"); + return false; + } + + void raw_unlock(const void * addr, size_t len) {} +#endif +}; + +// Replacement for std::vector that doesn't require zero-initialization. +struct gguf_buffer { + uint8_t * addr = NULL; + size_t size = 0; + + gguf_buffer() = default; + + void resize(size_t len) { +#ifdef GGML_USE_METAL + free(addr); + int result = posix_memalign((void **) &addr, getpagesize(), len); + if (result == 0) { + memset(addr, 0, len); + } + else { + addr = NULL; + } +#else + delete[] addr; + addr = new uint8_t[len]; +#endif + size = len; + } + + ~gguf_buffer() { +#ifdef GGML_USE_METAL + free(addr); +#else + delete[] addr; +#endif + addr = NULL; + } + + // disable copy and move + gguf_buffer(const gguf_buffer&) = delete; + gguf_buffer(gguf_buffer&&) = delete; + gguf_buffer& operator=(const gguf_buffer&) = delete; + gguf_buffer& operator=(gguf_buffer&&) = delete; +}; + +#ifdef GGML_USE_CUBLAS +#include "ggml-cuda.h" +struct gguf_ctx_buffer { + uint8_t * addr = NULL; + bool is_cuda; + size_t size = 0; + + gguf_ctx_buffer() = default; + + void resize(size_t size) { + free(); + + addr = (uint8_t *) ggml_cuda_host_malloc(size); + if (addr) { + is_cuda = true; + } + else { + // fall back to pageable memory + addr = new uint8_t[size]; + is_cuda = false; + } + this->size = size; + } + + void free() { + if (addr) { + if (is_cuda) { + ggml_cuda_host_free(addr); + } + else { + delete[] addr; + } + } + addr = NULL; + } + + ~gguf_ctx_buffer() { + free(); + } + + // disable copy and move + gguf_ctx_buffer(const gguf_ctx_buffer&) = delete; + gguf_ctx_buffer(gguf_ctx_buffer&&) = delete; + gguf_ctx_buffer& operator=(const gguf_ctx_buffer&) = delete; + gguf_ctx_buffer& operator=(gguf_ctx_buffer&&) = delete; +}; +#else +typedef gguf_buffer gguf_ctx_buffer; +#endif + +#endif diff --git a/gguf.py b/gguf.py index 18f42267aa78d..93591d5c76fa9 100644 --- a/gguf.py +++ b/gguf.py @@ -1,339 +1,339 @@ -"""TODOs -1. Implement writers for known architectures, LLaMA in particular. -2. Add docstrings from the format specs. -3. After development is done, Convert it to a proper pip-installable Python package, and possibly move it to its own repo under ggml-org. -""" - -import struct -import constants -from enum import IntEnum -from typing import Any, IO, List - -import numpy as np -import sys - - -class GGMLQuantizationType(IntEnum): - F32 = 0 - F16 = 1 - - -class GGUFValueType(IntEnum): - UINT8 = 0 - INT8 = 1 - UINT16 = 2 - INT16 = 3 - UINT32 = 4 - INT32 = 5 - FLOAT32 = 6 - BOOL = 7 - STRING = 8 - ARRAY = 9 - - @staticmethod - def get_type(val): - if isinstance(val, str) or isinstance(val, bytes) or isinstance(val, bytearray): - return GGUFValueType.STRING - elif isinstance(val, list): - return GGUFValueType.ARRAY - elif isinstance(val, float): - return GGUFValueType.FLOAT32 - elif isinstance(val, bool): - return GGUFValueType.BOOL - elif isinstance(val, int): - return GGUFValueType.INT32 - else: - print("Unknown type: "+str(type(val))) - sys.exit() - - -class GGUFWriter: - def __init__(self, fout: IO): - self.fout = fout - self.offset_tensor = 0 - self.data_alignment = constants.GGUF_DEFAULT_ALIGNMENT - self.kv_data = b"" - self.kv_data_count = 0 - self.ti_data = b"" - self.ti_data_count = 0 - - def write_header_to_file(self): - self.fout.write(struct.pack(" "GGUFWriter": - f = open(path, "wb") - return cls(f) - - def add_key(self, key: str): - self.add_val(key, GGUFValueType.STRING, add_vtype=False) - - def add_uint8(self, key: str, val: int): - self.add_key(key) - self.add_val(val, GGUFValueType.UINT8) - - def add_int8(self, key: str, val: int): - self.add_key(key) - self.add_val(val, GGUFValueType.INT8) - - def add_uint16(self, key: str, val: int): - self.add_key(key) - self.add_val(val, GGUFValueType.UINT16) - - def add_int16(self, key: str, val: int): - self.add_key(key) - self.add_val(val, GGUFValueType.INT16) - - def add_uint32(self, key: str, val: int): - self.add_key(key) - self.add_val(val, GGUFValueType.UINT32) - - def add_int32(self, key: str, val: int): - self.add_key(key) - self.add_val(val, GGUFValueType.INT32) - - def add_float32(self, key: str, val: float): - self.add_key(key) - self.add_val(val, GGUFValueType.FLOAT32) - - def add_bool(self, key: str, val: bool): - self.add_key(key) - self.add_val(val, GGUFValueType.BOOL) - - def add_string(self, key: str, val: str): - if len(val) == 0: return - self.add_key(key) - self.add_val(val, GGUFValueType.STRING) - - def add_array(self, key: str, val: list): - if not isinstance(val, list): - raise ValueError("Value must be a list for array type") - - self.add_key(key) - self.add_val(val, GGUFValueType.ARRAY) - - def add_val(self: str, val: Any, vtype: GGUFValueType = None, add_vtype: bool = True): - if vtype is None: - vtype = GGUFValueType.get_type(val) - - if add_vtype: - self.kv_data += struct.pack(" int: - return ((x + n - 1) // n) * n - - def add_tensor_info(self, name: str, tensor_shape: np.ndarray, tensor_dtype: np.dtype, tensor_nbytes: int): - encoded_name = name.encode("utf8") - self.ti_data += struct.pack(" "GGUFWriter": + f = open(path, "wb") + return cls(f) + + def add_key(self, key: str): + self.add_val(key, GGUFValueType.STRING, add_vtype=False) + + def add_uint8(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.UINT8) + + def add_int8(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.INT8) + + def add_uint16(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.UINT16) + + def add_int16(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.INT16) + + def add_uint32(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.UINT32) + + def add_int32(self, key: str, val: int): + self.add_key(key) + self.add_val(val, GGUFValueType.INT32) + + def add_float32(self, key: str, val: float): + self.add_key(key) + self.add_val(val, GGUFValueType.FLOAT32) + + def add_bool(self, key: str, val: bool): + self.add_key(key) + self.add_val(val, GGUFValueType.BOOL) + + def add_string(self, key: str, val: str): + if len(val) == 0: return + self.add_key(key) + self.add_val(val, GGUFValueType.STRING) + + def add_array(self, key: str, val: list): + if not isinstance(val, list): + raise ValueError("Value must be a list for array type") + + self.add_key(key) + self.add_val(val, GGUFValueType.ARRAY) + + def add_val(self: str, val: Any, vtype: GGUFValueType = None, add_vtype: bool = True): + if vtype is None: + vtype = GGUFValueType.get_type(val) + + if add_vtype: + self.kv_data += struct.pack(" int: + return ((x + n - 1) // n) * n + + def add_tensor_info(self, name: str, tensor_shape: np.ndarray, tensor_dtype: np.dtype, tensor_nbytes: int): + encoded_name = name.encode("utf8") + self.ti_data += struct.pack(" Date: Mon, 14 Aug 2023 13:51:09 +0200 Subject: [PATCH 143/242] Create convert-llama-7b-pth-to-gguf.py --- convert-llama-7b-pth-to-gguf.py | 302 ++++++++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100644 convert-llama-7b-pth-to-gguf.py diff --git a/convert-llama-7b-pth-to-gguf.py b/convert-llama-7b-pth-to-gguf.py new file mode 100644 index 0000000000000..d763503564c1d --- /dev/null +++ b/convert-llama-7b-pth-to-gguf.py @@ -0,0 +1,302 @@ +# 7b pth llama --> gguf conversion, GQA/70b not supported +# Only models with a single datafile are supported, like 7B +# HF files required in the model dir: config.json tokenizer_config.json tokenizer.json tokenizer.model + +import gguf +import gguf_namemap as tmap +import os +import sys +import struct +import json +import numpy as np +import torch +from typing import Any, List +from pathlib import Path +from sentencepiece import SentencePieceProcessor + + +#NDArray = np.ndarray[Any, Any] +# compatible with python < 3.9 +NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]' + +def count_model_parts(dir_model: str) -> int: + num_parts = 0 + for filename in os.listdir(dir_model): + if filename.startswith("consolidated."): + num_parts += 1 + + if num_parts > 0: + print("gguf: found " + str(num_parts) + " model parts") + return num_parts + +if len(sys.argv) < 3: + print("Usage: convert-h5-to-ggml.py dir-model ftype\n") + print(" ftype == 0 -> float32") + print(" ftype == 1 -> float16") + sys.exit(1) + + +# output in the same directory as the model +dir_model = sys.argv[1] +last_dir = os.path.basename(os.path.normpath(dir_model)) + + +# possible tensor data types +# ftype == 0 -> float32 +# ftype == 1 -> float16 +# +# map from ftype to string +ftype_str = ["f32", "f16"] + +ftype = 1 +if len(sys.argv) > 2: + ftype = int(sys.argv[2]) + if ftype < 0 or ftype > 1: + print("Invalid ftype: " + str(ftype)) + sys.exit(1) + +fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".gguf" + +print("gguf: loading model "+last_dir) + +with open(dir_model + "/config.json", "r", encoding="utf-8") as f: + hparams = json.load(f) + +if hparams["architectures"][0] != "LlamaForCausalLM": + print("Model architecture not supported: " + hparams["architectures"][0]) + sys.exit() + +# get number of model parts +num_parts = count_model_parts(dir_model) + +if num_parts > 1: + print("gguf: Only models with a single datafile are supported.") + sys.exit() + +gguf_writer = gguf.GGUFWriter.open(fname_out) + + +print("gguf: get model metadata") + +llm_arch = "llama" +block_count = hparams["num_hidden_layers"] +head_count = hparams["num_attention_heads"] + +if "num_key_value_heads" in hparams: + head_count_kv = hparams["num_key_value_heads"] +else: + head_count_kv = head_count + +if "_name_or_path" in hparams: + hf_repo = hparams["_name_or_path"] +else: + hf_repo="" + +gguf_writer.add_architecture(llm_arch) +gguf_writer.add_name(last_dir) +gguf_writer.add_file_type( "All tensors F32" if ftype == 0 else "Most tensors F16, some F32") +gguf_writer.add_source_hf_repo(hf_repo) +gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"]) +gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"]) +gguf_writer.add_block_count(llm_arch, block_count) +gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"]) +gguf_writer.add_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"]) +gguf_writer.add_head_count(llm_arch, head_count) +gguf_writer.add_head_count_kv(llm_arch, head_count_kv) +gguf_writer.add_layer_norm_rms_eps(llm_arch, hparams["rms_norm_eps"]) + + +# TOKENIZATION + +print("gguf: get tokenizer metadata") + +tokens: List[str] = [] +scores: List[float] = [] + +if Path(dir_model + "/tokenizer.model").is_file(): + # vocab type sentencepiece + print("gguf: get sentencepiece tokenizer vocab and scores") + + tokenizer = SentencePieceProcessor(dir_model + "/tokenizer.model") + + for i in range(tokenizer.vocab_size()): + text: bytes + if tokenizer.is_unknown(i): + text = " \u2047 ".encode("utf-8") + elif tokenizer.is_control(i): + text = b"" + if tokenizer.is_byte(i): + piece = tokenizer.id_to_piece(i) + if len(piece) != 6: + raise Exception(f"Invalid token: {piece}") + byte_value = int(piece[3:-1], 16) + text = struct.pack("B", byte_value) + else: + text = tokenizer.id_to_piece(i).replace("\u2581", " ").encode("utf-8") + score: float = tokenizer.get_score(i) + + tokens.append(text) + scores.append(score) + + gguf_writer.add_tokenizer_model("llama") + gguf_writer.add_token_list(tokens) + gguf_writer.add_token_scores(scores) + +if Path(dir_model + "/tokenizer.json").is_file(): + with open(dir_model + "/tokenizer.json", "r", encoding="utf-8") as f: + tokenizer = json.load(f) + + if "added_tokens" in tokenizer and Path(dir_model + "/tokenizer_config.json").is_file(): + print("gguf: get special token ids") + + with open(dir_model + "/tokenizer_config.json", "r", encoding="utf-8") as f: + tokenizer_config = json.load(f) + + # find special token ids + + if "bos_token" in tokenizer_config and tokenizer_config["bos_token"] != None: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["bos_token"]["content"]: + gguf_writer.add_bos_token_id(key["id"]) + + if "eos_token" in tokenizer_config and tokenizer_config["eos_token"] != None: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["eos_token"]["content"]: + gguf_writer.add_eos_token_id(key["id"]) + + if "unk_token" in tokenizer_config and tokenizer_config["unk_token"] != None: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["unk_token"]["content"]: + gguf_writer.add_unk_token_id(key["id"]) + + if "sep_token" in tokenizer_config and tokenizer_config["sep_token"] != None: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["sep_token"]["content"]: + gguf_writer.add_sep_token_id(key["id"]) + + if "pad_token" in tokenizer_config and tokenizer_config["pad_token"] != None: + for key in tokenizer["added_tokens"]: + if key["content"] == tokenizer_config["pad_token"]["content"]: + gguf_writer.add_pad_token_id(key["id"]) + + +# TENSORS + +tensor_map = tmap.get_tensor_namemap(block_count) + +# tensor info +print("gguf: get tensor metadata") + +part_names = ( f"consolidated.{n:02}.pth" for n in range(0, num_parts) ) + +for part_name in part_names: + print("gguf: loading model part '"+ part_name + "'") + model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu") + + for name in model_part.keys(): + data = model_part[name] + + # we don't need these + if name == "rope.freqs": + continue + + # convert any unsupported data types to float32 + if data.dtype != torch.float16 and data.dtype != torch.float32: + data = data.to(torch.float32) + + data = data.squeeze().numpy() + + # map tensor names + if name.endswith(".weight") and name[:-7] in tensor_map: + name = tensor_map[name[:-7]] + ".weight" + elif name.endswith(".bias") and name[:-5] in tensor_map: + name = tensor_map[name[:-5]] + ".bias" + else: + print( "Can not map tensor '" + name + "'" ) + sys.exit() + + n_dims = len(data.shape) + data_dtype = data.dtype + + # if f32 desired, convert any float16 to float32 + if ftype == 0 and data.dtype == np.float16: + data_dtype = np.float32 + + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if ftype == 1 and data_dtype == np.float16 and n_dims == 1: + data_dtype = np.float32 + + # if f16 desired, convert any float32 2-dim weight tensors to float16 + if ftype == 1 and data.dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data_dtype = np.float16 + + data_nbytes = data.size * 2 if data_dtype == np.float16 else data.size * 4 + + gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes) + + +print("gguf: write header") +gguf_writer.write_header_to_file() +print("gguf: write metadata") +gguf_writer.write_kv_data_to_file() +print("gguf: write tensor metadata") +gguf_writer.write_ti_data_to_file() + +# tensor data +print("gguf: convert and write tensor data") + +part_names = ( f"consolidated.{n:02}.pth" for n in range(0, num_parts) ) + +for part_name in part_names: + print("gguf: loading model part '"+ part_name + "'") + model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu") + + for name in model_part.keys(): + data = model_part[name] + + + old_dtype = data.dtype + + # we don't need these + if name == "rope.freqs": + continue + + # convert any unsupported data types to float32 + if data.dtype != torch.float16 and data.dtype != torch.float32: + data = data.to(torch.float32) + + data = data.squeeze().numpy() + + # map tensor names + if name.endswith(".weight") and name[:-7] in tensor_map: + name = tensor_map[name[:-7]] + ".weight" + elif name.endswith(".bias") and name[:-5] in tensor_map: + name = tensor_map[name[:-5]] + ".bias" + else: + print( "Can not map tensor '" + name + "'" ) + sys.exit() + + n_dims = len(data.shape) + data_dtype = data.dtype + + # if f32 desired, convert any float16 to float32 + if ftype == 0 and data.dtype == np.float16: + data = data.astype(np.float32) + + # TODO: Why cant we use these float16 as-is? There should be not reason to store float16 as float32 + if ftype == 1 and data_dtype == np.float16 and n_dims == 1: + data = data.astype(np.float32) + + # if f16 desired, convert any float32 2-dim weight tensors to float16 + if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2: + data = data.astype(np.float16) + + print( name + ", shape " + str(len(data.shape)) + ", " + str(old_dtype) + " --> " + str(data.dtype)) + + gguf_writer.write_tensor_to_file(data) + +gguf_writer.close() + + +print("gguf: model successfully exported to '" + fname_out + "'") +print("") From f00780b2ee99955fec8c02cd9f90aa230c9e026d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 14 Aug 2023 16:28:44 +0300 Subject: [PATCH 144/242] llama : sync gguf-llama.cpp with latest llama.cpp (#2608) * llama : sync gguf-llama.cpp with latest llama.cpp * minor : indentation + assert * llama : refactor gguf_buffer and gguf_ctx_buffer * llama : minor --- examples/gguf/gguf.cpp | 23 +- ggml-metal.h | 3 + ggml-metal.m | 15 + gguf-llama.cpp | 981 ++++++++++++++++++++++++++--------------- gguf-llama.h | 28 +- gguf-util.h | 97 ---- 6 files changed, 688 insertions(+), 459 deletions(-) diff --git a/examples/gguf/gguf.cpp b/examples/gguf/gguf.cpp index 6f454a2047ddd..b32367f301ee9 100644 --- a/examples/gguf/gguf.cpp +++ b/examples/gguf/gguf.cpp @@ -8,14 +8,19 @@ #include #include #include -/* + +#undef MIN +#undef MAX +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) > (b) ? (a) : (b)) + template static std::string to_string(const T & val) { std::stringstream ss; ss << val; return ss.str(); } -*/ + void gguf_ex_write_str(std::ofstream & fout, const std::string & val) { const int32_t n = val.size(); fout.write((const char *) &n, sizeof(n)); @@ -377,28 +382,28 @@ bool gguf_ex_read_2(const std::string & fname) { struct gguf_file file(fname.c_str(), "rb"); gguf_mmap data_mmap(&file, 0, false); + const int n_tensors = gguf_get_n_tensors(ctx); for (int i = 0; i < n_tensors; ++i) { - const char * name = gguf_get_tensor_name(ctx, i); - const size_t offset = gguf_get_data_offset(ctx) + gguf_get_tensor_offset(ctx, i); + const char * name = gguf_get_tensor_name(ctx, i); + const size_t offset = gguf_get_data_offset(ctx) + gguf_get_tensor_offset(ctx, i); + struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); cur->data = static_cast(data_mmap.addr) + offset; // print first 10 elements - const float * data = (const float *) cur->data; + const float * data = (const float *) cur->data; printf("%s data[:10] : ", name); - - for (int j = 0; j < 10; ++j) { + for (int j = 0; j < MIN(10, ggml_nelements(cur)); ++j) { printf("%f ", data[j]); } - printf("\n\n"); } -fprintf(stdout, "%s: ctx_data size: %zu\n", __func__, ggml_get_mem_size(ctx_data)); + fprintf(stdout, "%s: ctx_data size: %zu\n", __func__, ggml_get_mem_size(ctx_data)); ggml_free(ctx_data); gguf_free(ctx); diff --git a/ggml-metal.h b/ggml-metal.h index 16f1a0caacfac..1a5d96c339913 100644 --- a/ggml-metal.h +++ b/ggml-metal.h @@ -38,6 +38,9 @@ struct ggml_metal_context; struct ggml_metal_context * ggml_metal_init(int n_cb); void ggml_metal_free(struct ggml_metal_context * ctx); +void * ggml_metal_host_malloc(size_t n); +void ggml_metal_host_free (void * data); + // set the number of command buffers to use void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb); diff --git a/ggml-metal.m b/ggml-metal.m index b47a98e214b61..4e4491414ef20 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -224,6 +224,21 @@ void ggml_metal_free(struct ggml_metal_context * ctx) { free(ctx); } +void * ggml_metal_host_malloc(size_t n) { + void * data = NULL; + const int result = posix_memalign((void **) &data, getpagesize(), n); + if (result != 0) { + fprintf(stderr, "%s: error: posix_memalign failed\n", __func__); + return NULL; + } + + return data; +} + +void ggml_metal_host_free(void * data) { + free(data); +} + void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb) { ctx->n_cb = n_cb; } diff --git a/gguf-llama.cpp b/gguf-llama.cpp index 99da3c56dc92c..0f8eb3c902b1f 100644 --- a/gguf-llama.cpp +++ b/gguf-llama.cpp @@ -47,7 +47,6 @@ #include #include #include -#include #include #include #include @@ -56,28 +55,75 @@ #pragma warning(disable: 4244 4267) // possible loss of data #endif +static void llama_log_internal(llama_log_level level, const char* format, ...); +static void llama_log_callback_default(llama_log_level level, const char * text, void * user_data); +#define LLAMA_LOG_INFO(...) llama_log_internal(LLAMA_LOG_LEVEL_INFO , __VA_ARGS__) +#define LLAMA_LOG_WARN(...) llama_log_internal(LLAMA_LOG_LEVEL_WARN , __VA_ARGS__) +#define LLAMA_LOG_ERROR(...) llama_log_internal(LLAMA_LOG_LEVEL_ERROR, __VA_ARGS__) + +template +static std::string to_string(const T & val) { + std::stringstream ss; + ss << val; + return ss.str(); +} + +#if !defined(GGML_USE_CUBLAS) && !defined(GGML_USE_METAL) +#include "ggml-alloc.h" +#define LLAMA_USE_ALLOCATOR +#else #define LLAMA_USE_SCRATCH #define LLAMA_MAX_SCRATCH_BUFFERS 16 +#endif -// available llama models -enum e_model { - MODEL_UNKNOWN, - MODEL_3B, - MODEL_7B, - MODEL_13B, - MODEL_30B, - MODEL_65B, - MODEL_70B, -}; +typedef void (*offload_func_t)(struct ggml_tensor * tensor); -static const size_t kB = 1024; -static const size_t MB = 1024*1024; +#ifdef GGML_USE_CUBLAS +#define llama_host_malloc(n) ggml_cuda_host_malloc(n) +#define llama_host_free(data) ggml_cuda_host_free(data) +#elif GGML_USE_METAL +#define llama_host_malloc(n) ggml_metal_host_malloc(n) +#define llama_host_free(data) ggml_metal_host_free(data) +#else +#define llama_host_malloc(n) malloc(n) +#define llama_host_free(data) free(data) +#endif -// computed for n_ctx == 2048 -// TODO: dynamically determine these sizes -// needs modifications in ggml +struct llama_buffer { + void * data = NULL; + size_t size = 0; -typedef void (*offload_func_t)(struct ggml_tensor * tensor); + // fallback to malloc / free + // useful in cases where CUDA can try to allocate PINNED memory + bool fallback = false; + + void resize(size_t n) { + llama_host_free(data); + + data = llama_host_malloc(n); + if (!data) { + fallback = true; + data = malloc(n); + } else { + fallback = false; + } + + GGML_ASSERT(data); + size = n; + } + + ~llama_buffer() { + if (data) { + if (fallback) { // NOLINT + free(data); + } else { + llama_host_free(data); + } + } + + data = NULL; + } +}; void llama_nop(struct ggml_tensor * tensor) { // don't offload by default (void) tensor; @@ -102,6 +148,24 @@ static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * // memory sizes (calculated for n_batch == 512) // +// computed for n_ctx == 2048 +// TODO: dynamically determine these sizes +// needs modifications in ggml + +// available llama models +enum e_model { + MODEL_UNKNOWN, + MODEL_3B, + MODEL_7B, + MODEL_13B, + MODEL_30B, + MODEL_65B, + MODEL_70B, +}; + +static const size_t kB = 1024; +static const size_t MB = 1024*1024; + static const std::map & MEM_REQ_SCRATCH0(int n_ctx) { static std::map k_sizes = { @@ -143,7 +207,7 @@ static const std::map & MEM_REQ_EVAL() } // amount of VRAM needed per batch size to hold temporary results -// the values for 3b and 65b are not derived from testing but instead chosen conservatively +// the values for 3b are not derived from testing but instead chosen conservatively static const std::map & VRAM_REQ_SCRATCH_BASE() { static std::map k_sizes = { @@ -151,14 +215,14 @@ static const std::map & VRAM_REQ_SCRATCH_BASE() { MODEL_7B, 512ull * kB }, { MODEL_13B, 640ull * kB }, { MODEL_30B, 768ull * kB }, - { MODEL_65B, 1536ull * kB }, - { MODEL_70B, 1536ull * kB }, // TODO (likely can be reduced) + { MODEL_65B, 1280ull * kB }, + { MODEL_70B, 1280ull * kB }, }; return k_sizes; } // amount of VRAM needed per batch size and context to hold temporary results -// the values for 3b and 65b are not derived from testing but instead chosen conservatively +// the values for 3b are not derived from testing but instead chosen conservatively static const std::map & VRAM_REQ_SCRATCH_PER_CONTEXT() { static std::map k_sizes = { @@ -166,8 +230,8 @@ static const std::map & VRAM_REQ_SCRATCH_PER_CONTEXT() { MODEL_7B, 128ull }, { MODEL_13B, 160ull }, { MODEL_30B, 208ull }, - { MODEL_65B, 416ull }, - { MODEL_70B, 416ull }, // TODO (likely can be reduced) + { MODEL_65B, 256ull }, + { MODEL_70B, 256ull }, }; return k_sizes; } @@ -175,15 +239,15 @@ static const std::map & VRAM_REQ_SCRATCH_PER_CONTEXT() // default hparams (LLaMA 7B) struct llama_hparams { uint32_t n_vocab = 32000; - uint32_t n_ctx = 512; // this is provided as user input? + uint32_t n_ctx = 512; uint32_t n_embd = 4096; uint32_t n_head = 32; uint32_t n_head_kv = 32; uint32_t n_layer = 32; uint32_t n_rot = 64; - uint32_t n_ff = 11008; + uint32_t n_ff = 11008; - float f_rms_norm_eps = LLAMA_DEFAULT_RMS_EPS; + float f_rms_norm_eps = 1e-5; float rope_freq_base = 10000.0f; float rope_freq_scale = 1.0f; @@ -241,7 +305,7 @@ struct llama_kv_cache { struct ggml_context * ctx = NULL; - gguf_ctx_buffer buf; + llama_buffer buf; int n; // number of tokens currently in the cache @@ -292,7 +356,7 @@ struct llama_model { struct ggml_context * ctx = NULL; // the model memory buffer - gguf_ctx_buffer buf; + llama_buffer buf; // model memory mapped file std::unique_ptr mapping; @@ -329,13 +393,22 @@ struct llama_model { struct llama_context { llama_context(const llama_model & model) : model(model), t_load_us(model.t_load_us), t_start_us(model.t_start_us) {} -#ifdef GGML_USE_METAL ~llama_context() { + if (model_owner) { + delete &model; + } +#ifdef GGML_USE_METAL if (ctx_metal) { ggml_metal_free(ctx_metal); } - } #endif +#ifdef LLAMA_USE_ALLOCATOR + if (alloc) { + ggml_allocr_free(alloc); + } +#endif + } + std::mt19937 rng; bool has_evaluated_once = false; @@ -372,8 +445,19 @@ struct llama_context { // memory buffers used to evaluate the model // TODO: move in llama_state - gguf_ctx_buffer buf_compute; - gguf_ctx_buffer buf_scratch[LLAMA_MAX_SCRATCH_BUFFERS]; + llama_buffer buf_compute; + +#ifdef LLAMA_USE_ALLOCATOR + llama_buffer buf_alloc; + ggml_allocr * alloc = NULL; +#endif + +#ifdef LLAMA_USE_SCRATCH + llama_buffer buf_scratch[LLAMA_MAX_SCRATCH_BUFFERS]; + + int buf_last = 0; + size_t buf_max_size[LLAMA_MAX_SCRATCH_BUFFERS] = { 0 }; +#endif #ifdef GGML_USE_METAL ggml_metal_context * ctx_metal = NULL; @@ -383,10 +467,7 @@ struct llama_context { ggml_mpi_context * ctx_mpi = NULL; #endif - int buf_last = 0; - size_t buf_max_size[LLAMA_MAX_SCRATCH_BUFFERS] = { 0 }; - - void use_buf(struct ggml_context * ctx, int i) { + static void use_buf(struct ggml_context * ctx, int i) { #if defined(LLAMA_USE_SCRATCH) size_t last_size = 0; @@ -394,7 +475,7 @@ struct llama_context { last_size = ggml_set_scratch(ctx, { 0, 0, nullptr, }); } else { auto & buf = buf_scratch[i]; - last_size = ggml_set_scratch(ctx, { 0, buf.size, buf.addr, }); + last_size = ggml_set_scratch(ctx, { 0, buf.size, buf.data, }); } if (buf_last >= 0) { @@ -408,7 +489,7 @@ struct llama_context { #endif } - size_t get_buf_max_mem(int i) const { + static size_t get_buf_max_mem(int i) { #if defined(LLAMA_USE_SCRATCH) return buf_max_size[i]; #else @@ -418,6 +499,14 @@ struct llama_context { } }; +struct llama_state { + // We save the log callback globally + llama_log_callback log_callback = llama_log_callback_default; + void * log_callback_user_data = nullptr; +}; +// global state +static llama_state g_state; + template static T checked_mul(T a, T b) { T ret = a * b; @@ -470,17 +559,16 @@ struct gguf_load_tensors_map { enum gguf_file_version { GGUF_FILE_VERSION_V1 = 1, - }; - struct gguf_file_loader { gguf_file file; gguf_context * gguf_ctx; gguf_file_version file_version; llama_hparams hparams; llama_vocab vocab; -struct ggml_context * ctx_data = NULL; + + struct ggml_context * ctx_data = NULL; gguf_file_loader(const char * fname, gguf_load_tensors_map & tensors_map) : file(fname, "rb") { @@ -499,7 +587,7 @@ struct ggml_context * ctx_data = NULL; read_tensor_metadata(tensors_map); } - uint32_t read_u32(const char * key) { + uint32_t read_u32(const char * key) const { int i = gguf_find_key(gguf_ctx, key); if (i == -1) { throw std::runtime_error(format("cannot find param with key %s\n", key)); @@ -508,7 +596,7 @@ struct ggml_context * ctx_data = NULL; return gguf_get_val_u32(gguf_ctx, i); } - float read_f32(const char * key) { + float read_f32(const char * key) const { int i = gguf_find_key(gguf_ctx, key); if (i == -1) { throw std::runtime_error(format("cannot find param with key %s\n", key)); @@ -517,27 +605,26 @@ struct ggml_context * ctx_data = NULL; return gguf_get_val_f32(gguf_ctx, i); } - int read_n_vocab() { + int read_n_vocab() const { int i = gguf_find_key(gguf_ctx, "tokenizer.ggml.tokens"); - if (i == -1) { - throw std::runtime_error("cannot find token list in GGUF file\n"); - } + if (i == -1) { + throw std::runtime_error("cannot find token list in GGUF file\n"); + } - return gguf_get_arr_n(gguf_ctx, i); + return gguf_get_arr_n(gguf_ctx, i); } void read_hparams() { - // TODO define keys as constants in header // TODO: read all hparams from file - hparams.n_vocab = read_n_vocab(); - hparams.n_ctx = read_u32("llama.context_length"); - hparams.n_embd = read_u32("llama.embedding_length"); - hparams.n_ff = read_u32("llama.feed_forward_length"); - hparams.n_head = read_u32("llama.attention.head_count"); - hparams.n_layer = read_u32("llama.layer_count"); - hparams.n_rot = read_u32("llama.rope.dimension_count"); + hparams.n_vocab = read_n_vocab(); + hparams.n_ctx = read_u32("llama.context_length"); + hparams.n_embd = read_u32("llama.embedding_length"); + hparams.n_ff = read_u32("llama.feed_forward_length"); + hparams.n_head = read_u32("llama.attention.head_count"); + hparams.n_layer = read_u32("llama.layer_count"); + hparams.n_rot = read_u32("llama.rope.dimension_count"); hparams.f_rms_norm_eps = read_f32("llama.attention.layer_norm_rms_epsilon"); // LLaMAv2 @@ -568,7 +655,7 @@ struct ggml_context * ctx_data = NULL; } } - void read_tensor_metadata(gguf_load_tensors_map & tensors_map) { + void read_tensor_metadata(gguf_load_tensors_map & tensors_map) const { const int n_tensors = gguf_get_n_tensors(gguf_ctx); for (int i = 0; i < n_tensors; ++i) { @@ -576,16 +663,19 @@ struct ggml_context * ctx_data = NULL; const char * name = gguf_get_tensor_name(gguf_ctx, i); struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name); - uint32_t n_dims = cur->n_dims; + + const uint32_t n_dims = cur->n_dims; tensor.type = cur->type; tensor.ne.resize(n_dims); + for (uint32_t j = 0; j < n_dims; ++j) { - tensor.ne[j] = cur->ne[j]; + tensor.ne[j] = cur->ne[j]; } if (n_dims < 1 || n_dims > 2) { throw std::runtime_error(format("llama.cpp: tensor '%s' should not be %u-dimensional", name, n_dims)); } + switch (tensor.type) { case GGML_TYPE_F32: case GGML_TYPE_F16: @@ -605,7 +695,6 @@ struct ggml_context * ctx_data = NULL; } } - tensor.file_off = gguf_get_data_offset(gguf_ctx) + gguf_get_tensor_offset(gguf_ctx, i); tensor.name = name; @@ -632,47 +721,47 @@ struct gguf_file_saver { gguf_file_saver(const char * fname, gguf_file_loader * fl, enum llama_ftype new_ftype) : file(fname, "wb"), fl(fl) { - fprintf(stderr, "llama.cpp: saving model to %s\n", fname); - write_header(); - write_hparams(new_ftype); - } + fprintf(stderr, "llama.cpp: saving model to %s\n", fname); + write_header(); + write_hparams(new_ftype); + } void write_header() { const int32_t magic = GGUF_MAGIC; file.write_i32(magic); - const int32_t version = GGUF_VERSION; - file.write_i32(version); + const int32_t version = GGUF_VERSION; + file.write_i32(version); - const int32_t n_tensors = gguf_get_n_tensors(fl->gguf_ctx); - file.write_i32(n_tensors); + const int32_t n_tensors = gguf_get_n_tensors(fl->gguf_ctx); + file.write_i32(n_tensors); - const int32_t n_kv = gguf_get_n_kv(fl->gguf_ctx); - file.write_i32(n_kv); - } - - void write_hparam_arr_str(const std::string & key, enum gguf_type type, int i, int n_arr) { - std::vector data(n_arr); + const int32_t n_kv = gguf_get_n_kv(fl->gguf_ctx); + file.write_i32(n_kv); + } - for (int j = 0; j < n_arr; ++j) { - std::string val = gguf_get_arr_str(fl->gguf_ctx, i, j); - data[j] = val; - } + void write_hparam_arr_str(const std::string & key, enum gguf_type type, int i, int n_arr) { + std::vector data(n_arr); - file.write_arr(key, type, data); + for (int j = 0; j < n_arr; ++j) { + std::string val = gguf_get_arr_str(fl->gguf_ctx, i, j); + data[j] = val; } - void write_hparam_arr_f32(const std::string & key, enum gguf_type type, int i, int n_arr) { - std::vector data(n_arr); + file.write_arr(key, type, data); + } - for (int j = 0; j < n_arr; ++j) { - float val = gguf_get_arr_f32(fl->gguf_ctx, i, j); - data[j] = val; - } + void write_hparam_arr_f32(const std::string & key, enum gguf_type type, int i, int n_arr) { + std::vector data(n_arr); - file.write_arr(key, type, data); + for (int j = 0; j < n_arr; ++j) { + float val = gguf_get_arr_f32(fl->gguf_ctx, i, j); + data[j] = val; } + file.write_arr(key, type, data); + } + void write_hparams(enum llama_ftype new_ftype) { const int32_t n_kv = gguf_get_n_kv(fl->gguf_ctx); for (int i = 0; i < n_kv; ++i) { @@ -696,59 +785,62 @@ struct gguf_file_saver { switch(vtype) { case GGUF_TYPE_BOOL: - bool_val = gguf_get_val_bool(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_BOOL, bool_val); - break; + bool_val = gguf_get_val_bool(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_BOOL, bool_val); + break; case GGUF_TYPE_FLOAT32: - f32_val = gguf_get_val_f32(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_FLOAT32, f32_val); - break; + f32_val = gguf_get_val_f32(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_FLOAT32, f32_val); + break; case GGUF_TYPE_INT16: - i16_val = gguf_get_val_i16(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_INT16, i16_val); - break; + i16_val = gguf_get_val_i16(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_INT16, i16_val); + break; case GGUF_TYPE_INT32: - i32_val = gguf_get_val_i32(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_INT32, i32_val); - break; + i32_val = gguf_get_val_i32(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_INT32, i32_val); + break; case GGUF_TYPE_INT8: - i8_val = gguf_get_val_i8(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_INT8, i8_val); - break; + i8_val = gguf_get_val_i8(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_INT8, i8_val); + break; case GGUF_TYPE_STRING: - str_val = gguf_get_val_str(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_STRING, str_val); - break; + str_val = gguf_get_val_str(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_STRING, str_val); + break; case GGUF_TYPE_UINT16: - u16_val = gguf_get_val_u16(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_UINT16, u16_val); - break; + u16_val = gguf_get_val_u16(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_UINT16, u16_val); + break; case GGUF_TYPE_UINT32: - u32_val = gguf_get_val_u32(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_UINT32, u32_val); - break; + u32_val = gguf_get_val_u32(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_UINT32, u32_val); + break; case GGUF_TYPE_UINT8: - u8_val = gguf_get_val_u8(fl->gguf_ctx, i); - file.write_val(key, GGUF_TYPE_UINT8, u8_val); - break; + u8_val = gguf_get_val_u8(fl->gguf_ctx, i); + file.write_val(key, GGUF_TYPE_UINT8, u8_val); + break; case GGUF_TYPE_ARRAY: - arr_type = gguf_get_arr_type(fl->gguf_ctx, i); - n_arr = gguf_get_arr_n(fl->gguf_ctx, i); - if (arr_type == GGUF_TYPE_FLOAT32) { - write_hparam_arr_f32(key, arr_type, i, n_arr); + arr_type = gguf_get_arr_type(fl->gguf_ctx, i); + n_arr = gguf_get_arr_n(fl->gguf_ctx, i); + if (arr_type == GGUF_TYPE_FLOAT32) { + write_hparam_arr_f32(key, arr_type, i, n_arr); } else if (arr_type == GGUF_TYPE_STRING) { write_hparam_arr_str(key, GGUF_TYPE_STRING, i, n_arr); } else { throw std::runtime_error("not implemented"); } - break; + break; default: - throw std::runtime_error(format("cannot recognize value type for key %s\n", key)); + throw std::runtime_error(format("cannot recognize value type for key %s\n", key)); } } } - info_offset = file.tell(); + info_offset = file.tell(); + + GGML_ASSERT(gguf_get_data_offset(fl->gguf_ctx) >= info_offset); + size_t count = gguf_get_data_offset(fl->gguf_ctx) - info_offset; file.write_zeros(count); file.seek(info_offset, SEEK_SET); @@ -983,7 +1075,7 @@ static bool kv_cache_init( struct ggml_init_params params; params.mem_size = cache.buf.size; - params.mem_buffer = cache.buf.addr; + params.mem_buffer = cache.buf.data; params.no_alloc = false; cache.ctx = ggml_init(params); @@ -1016,8 +1108,6 @@ struct llama_context_params llama_context_default_params() { /*.seed =*/ LLAMA_DEFAULT_SEED, /*.n_ctx =*/ 512, /*.n_batch =*/ 512, - /*.n_gqa =*/ 1, - /*.rms_norm_eps =*/ LLAMA_DEFAULT_RMS_EPS, /*.gpu_layers =*/ 0, /*.main_gpu =*/ 0, /*.tensor_split =*/ nullptr, @@ -1026,6 +1116,7 @@ struct llama_context_params llama_context_default_params() { /*.progress_callback =*/ nullptr, /*.progress_callback_user_data =*/ nullptr, /*.low_vram =*/ false, + /*.mul_mat_q =*/ false, /*.f16_kv =*/ true, /*.logits_all =*/ false, /*.vocab_only =*/ false, @@ -1144,11 +1235,10 @@ static void llama_model_load_internal( llama_vocab & vocab, int n_ctx, int n_batch, - int n_gqa, - float rms_norm_eps, int n_gpu_layers, int main_gpu, const float * tensor_split, + const bool mul_mat_q, float rope_freq_base, float rope_freq_scale, bool low_vram, @@ -1158,8 +1248,6 @@ static void llama_model_load_internal( bool vocab_only, llama_progress_callback progress_callback, void * progress_callback_user_data) { - GGML_UNUSED(rms_norm_eps); // TODO: update function signature to remove this - model.t_start_us = ggml_time_us(); std::unique_ptr ml(new llama_model_loader(fname, use_mmap)); @@ -1189,11 +1277,15 @@ static void llama_model_load_internal( hparams.n_ctx = n_ctx; // LLaMAv2 - hparams.n_head_kv = hparams.n_head / n_gqa; - if (model.type == e_model::MODEL_65B && n_gqa == 8) { - fprintf(stderr, "%s: warning: assuming 70B model based on GQA == %d\n", __func__, n_gqa); - model.type = e_model::MODEL_70B; + // TODO: probably not needed + { + const auto n_gqa = hparams.n_gqa(); + + if (model.type == e_model::MODEL_65B && n_gqa == 8) { + fprintf(stderr, "%s: warning: assuming 70B model based on GQA == %d\n", __func__, n_gqa); + model.type = e_model::MODEL_70B; } + } hparams.rope_freq_base = rope_freq_base; hparams.rope_freq_scale = rope_freq_scale; @@ -1202,27 +1294,21 @@ static void llama_model_load_internal( const uint32_t n_ff = hparams.n_ff; { - fprintf(stderr, "%s: format = %s\n", __func__, gguf_file_version_name(file_version)); - fprintf(stderr, "%s: n_vocab = %u\n", __func__, hparams.n_vocab); - fprintf(stderr, "%s: n_ctx = %u\n", __func__, hparams.n_ctx); - fprintf(stderr, "%s: n_embd = %u\n", __func__, hparams.n_embd); - fprintf(stderr, "%s: n_head = %u\n", __func__, hparams.n_head); - fprintf(stderr, "%s: n_head_kv = %u\n", __func__, hparams.n_head_kv); - fprintf(stderr, "%s: n_layer = %u\n", __func__, hparams.n_layer); - fprintf(stderr, "%s: n_rot = %u\n", __func__, hparams.n_rot); // a.k.a. n_embd_head, n_head_dim - fprintf(stderr, "%s: n_gqa = %u\n", __func__, hparams.n_gqa()); - fprintf(stderr, "%s: rnorm_eps = %.1e\n", __func__, hparams.f_rms_norm_eps); - fprintf(stderr, "%s: n_ff = %u\n", __func__, n_ff); - fprintf(stderr, "%s: freq_base = %.1f\n", __func__, hparams.rope_freq_base); - fprintf(stderr, "%s: freq_scale = %g\n", __func__, hparams.rope_freq_scale); - fprintf(stderr, "%s: ftype = %u (%s)\n", __func__, hparams.ftype, llama_ftype_name(hparams.ftype)); - fprintf(stderr, "%s: model size = %s\n", __func__, llama_model_type_name(model.type)); - } - - if (hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || - hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_1 || - hparams.ftype == LLAMA_FTYPE_MOSTLY_Q8_0) { - throw std::runtime_error(format("this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1508)")); + LLAMA_LOG_INFO("%s: format = %s\n", __func__, gguf_file_version_name(file_version)); + LLAMA_LOG_INFO("%s: n_vocab = %u\n", __func__, hparams.n_vocab); + LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, hparams.n_ctx); + LLAMA_LOG_INFO("%s: n_embd = %u\n", __func__, hparams.n_embd); + LLAMA_LOG_INFO("%s: n_head = %u\n", __func__, hparams.n_head); + LLAMA_LOG_INFO("%s: n_head_kv = %u\n", __func__, hparams.n_head_kv); + LLAMA_LOG_INFO("%s: n_layer = %u\n", __func__, hparams.n_layer); + LLAMA_LOG_INFO("%s: n_rot = %u\n", __func__, hparams.n_rot); // a.k.a. n_embd_head, n_head_dim + LLAMA_LOG_INFO("%s: n_gqa = %u\n", __func__, hparams.n_gqa()); + LLAMA_LOG_INFO("%s: rnorm_eps = %.1e\n", __func__, hparams.f_rms_norm_eps); + LLAMA_LOG_INFO("%s: n_ff = %u\n", __func__, n_ff); + LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, hparams.rope_freq_base); + LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, hparams.rope_freq_scale); + LLAMA_LOG_INFO("%s: ftype = %u (%s)\n", __func__, hparams.ftype, llama_ftype_name(hparams.ftype)); + LLAMA_LOG_INFO("%s: model size = %s\n", __func__, llama_model_type_name(model.type)); } if (vocab_only) { @@ -1234,19 +1320,19 @@ static void llama_model_load_internal( size_t ctx_size; size_t mmapped_size; ml->calc_sizes(&ctx_size, &mmapped_size); - fprintf(stderr, "%s: ggml ctx size = %7.2f MB\n", __func__, ctx_size/1024.0/1024.0); + LLAMA_LOG_INFO("%s: ggml ctx size = %7.2f MB\n", __func__, ctx_size/1024.0/1024.0); // create the ggml context { model.buf.resize(ctx_size); if (use_mlock) { - model.mlock_buf.init (model.buf.addr); + model.mlock_buf.init (model.buf.data); model.mlock_buf.grow_to(model.buf.size); } struct ggml_init_params params = { /*.mem_size =*/ model.buf.size, - /*.mem_buffer =*/ model.buf.addr, + /*.mem_buffer =*/ model.buf.data, /*.no_alloc =*/ ml->use_mmap, }; @@ -1257,13 +1343,15 @@ static void llama_model_load_internal( } (void) main_gpu; + (void) mul_mat_q; #if defined(GGML_USE_CUBLAS) - fprintf(stderr, "%s: using CUDA for GPU acceleration\n", __func__); + LLAMA_LOG_INFO("%s: using CUDA for GPU acceleration\n", __func__); ggml_cuda_set_main_device(main_gpu); + ggml_cuda_set_mul_mat_q(mul_mat_q); #define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU #define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_GPU_SPLIT #elif defined(GGML_USE_CLBLAST) - fprintf(stderr, "%s: using OpenCL for GPU acceleration\n", __func__); + LLAMA_LOG_INFO("%s: using OpenCL for GPU acceleration\n", __func__); #define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU #define LLAMA_BACKEND_OFFLOAD_SPLIT GGML_BACKEND_GPU #else @@ -1353,25 +1441,29 @@ static void llama_model_load_internal( const size_t scale = memory_type == GGML_TYPE_F32 ? 2 : 1; // this is the total memory required to run the inference - const size_t mem_required = + size_t mem_required = ctx_size + - mmapped_size - vram_weights + // weights in VRAM not in memory + mmapped_size - vram_weights; // weights in VRAM not in memory + +#ifndef LLAMA_USE_ALLOCATOR + mem_required += MEM_REQ_SCRATCH0(hparams.n_ctx).at(model.type) + MEM_REQ_SCRATCH1().at(model.type) + MEM_REQ_EVAL().at(model.type); +#endif // this is the memory required by one llama_state const size_t mem_required_state = scale*hparams.kv_size(); - fprintf(stderr, "%s: mem required = %7.2f MB (+ %7.2f MB per state)\n", __func__, + LLAMA_LOG_INFO("%s: mem required = %7.2f MB (+ %7.2f MB per state)\n", __func__, mem_required / 1024.0 / 1024.0, mem_required_state / 1024.0 / 1024.0); (void) vram_scratch; (void) n_batch; #ifdef GGML_USE_CUBLAS if (low_vram) { - fprintf(stderr, "%s: not allocating a VRAM scratch buffer due to low VRAM option\n", __func__); + LLAMA_LOG_INFO("%s: not allocating a VRAM scratch buffer due to low VRAM option\n", __func__); ggml_cuda_set_scratch_size(0); // disable scratch } else { const size_t vram_scratch_base = VRAM_REQ_SCRATCH_BASE().at(model.type); @@ -1379,7 +1471,7 @@ static void llama_model_load_internal( vram_scratch = n_batch * (vram_scratch_base + n_ctx * vram_scratch_per_context); ggml_cuda_set_scratch_size(vram_scratch); if (n_gpu_layers > 0) { - fprintf(stderr, "%s: allocating batch_size x (%zd kB + n_ctx x %zd B) = %zd MB VRAM for the scratch buffer\n", + LLAMA_LOG_INFO("%s: allocating batch_size x (%zd kB + n_ctx x %zd B) = %zd MB VRAM for the scratch buffer\n", __func__, vram_scratch_base / kB, vram_scratch_per_context, (vram_scratch + MB - 1) / MB); // round up } @@ -1389,9 +1481,9 @@ static void llama_model_load_internal( #if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer)); - fprintf(stderr, "%s: offloading %d repeating layers to GPU\n", __func__, n_gpu); + LLAMA_LOG_INFO("%s: offloading %d repeating layers to GPU\n", __func__, n_gpu); if (n_gpu_layers > (int) hparams.n_layer) { - fprintf(stderr, "%s: offloading non-repeating layers to GPU\n", __func__); + LLAMA_LOG_INFO("%s: offloading non-repeating layers to GPU\n", __func__); } size_t vram_kv_cache = 0; @@ -1400,17 +1492,17 @@ static void llama_model_load_internal( const int max_offloadable_layers = low_vram ? hparams.n_layer + 1 : hparams.n_layer + 3; if (n_gpu_layers > (int) hparams.n_layer + 1) { if (low_vram) { - fprintf(stderr, "%s: cannot offload v cache to GPU due to low VRAM option\n", __func__); + LLAMA_LOG_INFO("%s: cannot offload v cache to GPU due to low VRAM option\n", __func__); } else { - fprintf(stderr, "%s: offloading v cache to GPU\n", __func__); + LLAMA_LOG_INFO("%s: offloading v cache to GPU\n", __func__); vram_kv_cache += hparams.kv_size() / 2; } } if (n_gpu_layers > (int) hparams.n_layer + 2) { if (low_vram) { - fprintf(stderr, "%s: cannot offload k cache to GPU due to low VRAM option\n", __func__); + LLAMA_LOG_WARN("%s: cannot offload k cache to GPU due to low VRAM option\n", __func__); } else { - fprintf(stderr, "%s: offloading k cache to GPU\n", __func__); + LLAMA_LOG_INFO("%s: offloading k cache to GPU\n", __func__); vram_kv_cache += hparams.kv_size() / 2; } } @@ -1419,9 +1511,9 @@ static void llama_model_load_internal( const int max_offloadable_layers = hparams.n_layer + 1; #endif // GGML_USE_CUBLAS - fprintf(stderr, "%s: offloaded %d/%d layers to GPU\n", + LLAMA_LOG_INFO("%s: offloaded %d/%d layers to GPU\n", __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers); - fprintf(stderr, "%s: total VRAM used: %zu MB\n", + LLAMA_LOG_INFO("%s: total VRAM used: %zu MB\n", __func__, (vram_weights + vram_scratch + vram_kv_cache + MB - 1) / MB); // round up #else (void) n_gpu_layers; @@ -1459,11 +1551,10 @@ static bool llama_model_load( llama_vocab & vocab, int n_ctx, int n_batch, - int n_gqa, - float rms_norm_eps, int n_gpu_layers, int main_gpu, const float * tensor_split, + const bool mul_mat_q, float rope_freq_base, float rope_freq_scale, bool low_vram, @@ -1474,41 +1565,25 @@ static bool llama_model_load( llama_progress_callback progress_callback, void *progress_callback_user_data) { try { - llama_model_load_internal(fname, model, vocab, n_ctx, n_batch, n_gqa, rms_norm_eps, n_gpu_layers, main_gpu, tensor_split, rope_freq_base, rope_freq_scale, low_vram, memory_type, + llama_model_load_internal(fname, model, vocab, n_ctx, n_batch, n_gpu_layers, + main_gpu, tensor_split, mul_mat_q, rope_freq_base, rope_freq_scale, low_vram, memory_type, use_mmap, use_mlock, vocab_only, progress_callback, progress_callback_user_data); return true; } catch (const std::exception & err) { - fprintf(stderr, "error loading model: %s\n", err.what()); + LLAMA_LOG_ERROR("error loading model: %s\n", err.what()); return false; } } -// evaluate the transformer -// -// - lctx: llama context -// - tokens: new batch of tokens to process -// - embd embeddings input -// - n_tokens number of tokens -// - n_past: the context size so far -// - n_threads: number of threads to use -// -static bool llama_eval_internal( +static struct ggml_cgraph * llama_build_graph( llama_context & lctx, const llama_token * tokens, const float * embd, int n_tokens, - int n_past, - int n_threads, - const char * cgraph_fname) { + int n_past) { GGML_ASSERT((!tokens && embd) || (tokens && !embd)); -#ifdef GGML_USE_MPI - ggml_mpi_eval_init(lctx.ctx_mpi, &n_tokens, &n_past, &n_threads); -#endif - - const int64_t t_start_us = ggml_time_us(); - const int N = n_tokens; const auto & model = lctx.model; @@ -1524,7 +1599,6 @@ static bool llama_eval_internal( const int64_t n_head = hparams.n_head; const int64_t n_head_kv = hparams.n_head_kv; const int64_t n_embd_head = hparams.n_embd_head(); - const int64_t n_vocab = hparams.n_vocab; const int64_t n_embd_gqa = hparams.n_embd_gqa(); @@ -1539,26 +1613,35 @@ static bool llama_eval_internal( auto & mem_per_token = lctx.mem_per_token; auto & buf_compute = lctx.buf_compute; + struct ggml_init_params params = { /*.mem_size =*/ buf_compute.size, - /*.mem_buffer =*/ buf_compute.addr, + /*.mem_buffer =*/ buf_compute.data, /*.no_alloc =*/ false, }; +#ifdef LLAMA_USE_ALLOCATOR + params.no_alloc = true; +#endif + struct ggml_context * ctx0 = ggml_init(params); ggml_cgraph * gf = ggml_new_graph(ctx0); - // for big prompts, if BLAS is enabled, it is better to use only one thread - // otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance - n_threads = N >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas() ? 1 : n_threads; - struct ggml_tensor * cur; struct ggml_tensor * inpL; if (tokens) { struct ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); + +#ifdef LLAMA_USE_ALLOCATOR + ggml_allocr_alloc(lctx.alloc, inp_tokens); + if (!ggml_allocr_is_measure(lctx.alloc)) { + memcpy(inp_tokens->data, tokens, N*ggml_element_size(inp_tokens)); + } +#else memcpy(inp_tokens->data, tokens, N*ggml_element_size(inp_tokens)); +#endif ggml_set_name(inp_tokens, "inp_tokens"); inpL = ggml_get_rows(ctx0, model.tok_embeddings, inp_tokens); @@ -1568,7 +1651,15 @@ static bool llama_eval_internal( #endif inpL = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N); + +#ifdef LLAMA_USE_ALLOCATOR + ggml_allocr_alloc(lctx.alloc, inpL); + if (!ggml_allocr_is_measure(lctx.alloc)) { + memcpy(inpL->data, embd, N * n_embd * ggml_element_size(inpL)); + } +#else memcpy(inpL->data, embd, N * n_embd * ggml_element_size(inpL)); +#endif } const int i_gpu_start = n_layer - n_gpu_layers; @@ -1595,6 +1686,17 @@ static bool llama_eval_internal( } #endif // GGML_USE_CUBLAS + struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1); +#ifdef LLAMA_USE_ALLOCATOR + ggml_allocr_alloc(lctx.alloc, KQ_scale); + if (!ggml_allocr_is_measure(lctx.alloc)) { + ggml_set_f32(KQ_scale, 1.0f/sqrtf(float(n_embd)/n_head)); + } +#else + ggml_set_f32(KQ_scale, 1.0f/sqrtf(float(n_embd)/n_head)); +#endif + ggml_set_name(KQ_scale, "1/sqrt(n_embd_head)"); + for (int il = 0; il < n_layer; ++il) { ggml_format_name(inpL, "layer_inp_%d", il); @@ -1690,9 +1792,6 @@ static bool llama_eval_internal( ggml_set_name(KQ, "KQ"); // KQ_scaled = KQ / sqrt(n_embd_head) - struct ggml_tensor * KQ_scale = ggml_new_f32(ctx0, 1.0f/sqrtf(float(n_embd)/n_head)); - ggml_set_name(KQ_scale, "1/sqrt(n_embd_head)"); - // KQ_scaled shape [n_past + N, N, n_head, 1] struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, KQ_scale); offload_func_kq(KQ_scaled); @@ -1808,9 +1907,6 @@ static bool llama_eval_internal( lctx.use_buf(ctx0, 0); - // used at the end to optionally extract the embeddings - struct ggml_tensor * embeddings = NULL; - // norm { cur = ggml_rms_norm(ctx0, inpL, rms_norm_eps); @@ -1821,8 +1917,6 @@ static bool llama_eval_internal( cur = ggml_mul(ctx0, cur, model.norm); // offload_func_nr(cur); // TODO CPU + GPU mirrored backend ggml_set_name(cur, "result_norm"); - - embeddings = cur; } // lm_head @@ -1834,23 +1928,103 @@ static bool llama_eval_internal( // logits -> probs //cur = ggml_soft_max_inplace(ctx0, cur); - // run the computation ggml_build_forward_expand(gf, cur); - // fprintf(stderr, "graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf.n_nodes, gf.n_leafs); + if (mem_per_token == 0) { + mem_per_token = ggml_used_mem(ctx0)/N; + } + +#if 0 + LLAMA_LOG_INFO("\n%s: used_mem: eval ctx %.3f MB, scratch %.3f MB %.3f MB, work buf %.3f MB, n_past = %d, N = %d\n", __func__, + ggml_used_mem(ctx0)/1024.0/1024.0, + lctx.get_buf_max_mem(0)/1024.0/1024.0, + lctx.get_buf_max_mem(1)/1024.0/1024.0, + lctx.work_buffer.size()/1024.0/1024.0, + n_past, N); +#endif + + ggml_free(ctx0); + + return gf; +} + +// evaluate the transformer +// +// - lctx: llama context +// - tokens: new batch of tokens to process +// - embd embeddings input +// - n_tokens number of tokens +// - n_past: the context size so far +// - n_threads: number of threads to use +// +static bool llama_eval_internal( + llama_context & lctx, + const llama_token * tokens, + const float * embd, + int n_tokens, + int n_past, + int n_threads, + const char * cgraph_fname) { + + GGML_ASSERT((!tokens && embd) || (tokens && !embd)); + + const int64_t t_start_us = ggml_time_us(); + +#ifdef GGML_USE_MPI + ggml_mpi_eval_init(lctx.ctx_mpi, &n_tokens, &n_past, &n_threads); +#endif + + const int N = n_tokens; + + const auto & model = lctx.model; + const auto & hparams = model.hparams; + + const auto & kv_self = lctx.kv_self; + + GGML_ASSERT(!!kv_self.ctx); + + const int64_t n_embd = hparams.n_embd; + const int64_t n_vocab = hparams.n_vocab; + +#ifdef LLAMA_USE_ALLOCATOR + ggml_allocr_reset(lctx.alloc); +#endif + + ggml_cgraph * gf = llama_build_graph(lctx, tokens, embd, n_tokens, n_past); + +#ifdef LLAMA_USE_ALLOCATOR + ggml_allocr_alloc_graph(lctx.alloc, gf); +#endif + + // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs); + + // for big prompts, if BLAS is enabled, it is better to use only one thread + // otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance + n_threads = N >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas() ? 1 : n_threads; + + struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; + struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 2]; + + GGML_ASSERT(strcmp(res->name, "result_output") == 0); + GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0); #if GGML_USE_MPI + const int64_t n_layer = hparams.n_layer; ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer); #endif #ifdef GGML_USE_METAL if (lctx.ctx_metal && N == 1) { - if (!ggml_metal_if_optimized(lctx.ctx_metal)) { - ggml_metal_graph_find_concurrency(lctx.ctx_metal, gf); - } + // TODO: disabled until #2413 is resolved + //if (!ggml_metal_if_optimized(lctx.ctx_metal)) { + // ggml_metal_graph_find_concurrency(lctx.ctx_metal, gf); + //} ggml_metal_set_n_cb (lctx.ctx_metal, n_threads); ggml_metal_graph_compute(lctx.ctx_metal, gf); - ggml_metal_get_tensor (lctx.ctx_metal, cur); + ggml_metal_get_tensor (lctx.ctx_metal, res); + if (!lctx.embedding.empty()) { + ggml_metal_get_tensor(lctx.ctx_metal, embeddings); + } } else { // IMPORTANT: // Since we don't have efficient Matrix x Matrix Metal multiplication yet, we fallback to vanilla @@ -1881,8 +2055,6 @@ static bool llama_eval_internal( // update kv token count lctx.kv_self.n = n_past + N; - struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1]; - if (cgraph_fname) { ggml_graph_export(gf, cgraph_fname); } @@ -1920,21 +2092,6 @@ static bool llama_eval_internal( memcpy(embedding_out.data(), (float *) ggml_get_data(embeddings) + (n_embd*(N - 1)), sizeof(float)*n_embd); } - if (mem_per_token == 0) { - mem_per_token = ggml_used_mem(ctx0)/N; - } - -#if 0 - printf("\n%s: used_mem: eval ctx %.3f MB, scratch %.3f MB %.3f MB, work buf %.3f MB, n_past = %d, N = %d\n", __func__, - ggml_used_mem(ctx0)/1024.0/1024.0, - lctx.get_buf_max_mem(0)/1024.0/1024.0, - lctx.get_buf_max_mem(1)/1024.0/1024.0, - lctx.work_buffer.size()/1024.0/1024.0, - n_past, N); -#endif - - ggml_free(ctx0); - // measure the performance only for the single-token evals if (N == 1) { lctx.t_eval_us += ggml_time_us() - t_start_us; @@ -2026,7 +2183,7 @@ struct llama_tokenizer { left_sym.n += right_sym.n; right_sym.n = 0; - //printf("left = '%*s' size = %zu\n", (int) left_sym.n, left_sym.text, bigram.size); + //LLAMA_LOG_INFO("left = '%*s' size = %zu\n", (int) left_sym.n, left_sym.text, bigram.size); // remove the right sym from the chain left_sym.next = right_sym.next; @@ -2046,7 +2203,9 @@ struct llama_tokenizer { if (token == vocab_.token_to_id.end()) { // output any symbols that did not form tokens as bytes. for (int j = 0; j < (int) symbol.n; ++j) { - llama_vocab::id token_id = static_cast(symbol.text[j]) + 3; + // NOTE: old version, before #2420 - not sure what are the implications of this + //llama_vocab::id token_id = static_cast(symbol.text[j]) + 3; + llama_vocab::id token_id = vocab_.token_to_id.at(std::string(1, symbol.text[j])); output.push_back(token_id); } } else { @@ -2904,11 +3063,11 @@ void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar // quantization // -static void llama_convert_tensor_internal(const gguf_load_tensor & tensor, gguf_buffer & output, const int nelements, const int nthread) { - if (output.size < nelements * sizeof(float)) { - output.resize(nelements * sizeof(float)); +static void llama_convert_tensor_internal(const gguf_load_tensor & tensor, std::vector & output, const size_t nelements, const int nthread) { + if (output.size() < nelements) { + output.resize(nelements); } - float * f32_output = (float *) output.addr; + float * f32_output = (float *) output.data(); ggml_type_traits_t qtype; if (ggml_is_quantized(tensor.type)) { @@ -3026,13 +3185,16 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s }; size_t idx = 0; + + std::vector read_data; + std::vector work; + for (gguf_load_tensor & tensor : model_loader->tensors_map.tensors) { - gguf_buffer read_data; read_data.resize(tensor.size); - tensor.data = read_data.addr; + tensor.data = read_data.data(); model_loader->load_data_for(tensor); - printf("[%4zu/%4zu] %36s - %16s, type = %6s, ", + LLAMA_LOG_INFO("[%4zu/%4zu] %36s - %16s, type = %6s, ", ++idx, model_loader->tensors_map.tensors.size(), tensor.name.c_str(), llama_format_tensor_shape(tensor.ne).c_str(), ggml_type_name(tensor.type)); @@ -3048,13 +3210,12 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s enum ggml_type new_type; void * new_data; size_t new_size; - gguf_buffer work; if (!quantize) { new_type = tensor.type; new_data = tensor.data; new_size = tensor.size; - printf("size = %8.3f MB\n", tensor.size/1024.0/1024.0); + LLAMA_LOG_INFO("size = %8.3f MB\n", tensor.size/1024.0/1024.0); } else { new_type = quantized_type; #ifdef GGML_USE_K_QUANTS @@ -3089,26 +3250,27 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s int nx = tensor.ne.at(0); int ny = tensor.ne.at(1); if (nx % QK_K != 0 || ny % QK_K != 0) { - fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K); + LLAMA_LOG_INFO("\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K); convert_incompatible_tensor = true; } } if (convert_incompatible_tensor) { if (tensor.name == "output.weight") { new_type = GGML_TYPE_F16; //fall back to F16 instead of just failing. - fprintf(stderr, "F16 will be used for this tensor instead.\n"); + LLAMA_LOG_WARN("F16 will be used for this tensor instead.\n"); } else if (tensor.name == "tok_embeddings.weight") { new_type = GGML_TYPE_Q4_0; //fall back to Q4_0 instead of just failing. - fprintf(stderr, "Q4_0 will be used for this tensor instead.\n"); + LLAMA_LOG_WARN("Q4_0 will be used for this tensor instead.\n"); } else { throw std::runtime_error("Unsupported tensor size encountered\n"); } } #endif + const size_t nelements = tensor.ne.at(0) * tensor.ne.at(1); + float * f32_data; - size_t nelements = tensor.ne.at(0) * tensor.ne.at(1); - gguf_buffer f32_conv_buf; + std::vector f32_conv_buf; if (tensor.type == GGML_TYPE_F32) { f32_data = (float *) tensor.data; @@ -3116,17 +3278,17 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s throw std::runtime_error(format("requantizing from type %s is disabled", ggml_type_name(tensor.type))); } else { llama_convert_tensor_internal(tensor, f32_conv_buf, nelements, nthread); - f32_data = (float *) f32_conv_buf.addr; + f32_data = (float *) f32_conv_buf.data(); } - printf("quantizing to %s .. ", ggml_type_name(new_type)); + LLAMA_LOG_INFO("quantizing to %s .. ", ggml_type_name(new_type)); fflush(stdout); work.resize(nelements * 4); // upper bound on size - new_data = work.addr; + new_data = work.data(); std::vector hist_cur(1 << 4, 0); - int chunk_size = 32 * 512; + const int chunk_size = 32 * 512; const int nchunk = (nelements + chunk_size - 1)/chunk_size; const int nthread_use = nthread > 1 ? std::max(1, std::min(nthread, nchunk)) : 1; if (nthread_use < 2) { @@ -3134,7 +3296,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } else { size_t counter = 0; new_size = 0; - auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, nelements, chunk_size] () { + auto compute = [&mutex, &counter, &hist_cur, &new_size, new_type, f32_data, new_data, nelements] () { std::vector local_hist; size_t local_size = 0; while (true) { @@ -3169,7 +3331,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } } - printf("size = %8.2f MB -> %8.2f MB | hist: ", tensor.size/1024.0/1024.0, new_size/1024.0/1024.0); + LLAMA_LOG_INFO("size = %8.2f MB -> %8.2f MB | hist: ", tensor.size/1024.0/1024.0, new_size/1024.0/1024.0); int64_t tot_count = 0; for (size_t i = 0; i < hist_cur.size(); i++) { hist_all[i] += hist_cur[i]; @@ -3178,18 +3340,18 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (tot_count > 0) { for (size_t i = 0; i < hist_cur.size(); i++) { - printf("%5.3f ", hist_cur[i] / float(nelements)); + LLAMA_LOG_INFO("%5.3f ", hist_cur[i] / float(nelements)); } } - printf("\n"); + LLAMA_LOG_INFO("\n"); } total_size_org += tensor.size; total_size_new += new_size; file_saver.write_tensor(tensor, new_type, new_data, new_size); } - printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0); - printf("%s: quant size = %8.2f MB\n", __func__, total_size_new/1024.0/1024.0); + LLAMA_LOG_INFO("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0); + LLAMA_LOG_INFO("%s: quant size = %8.2f MB\n", __func__, total_size_new/1024.0/1024.0); { int64_t sum_all = 0; @@ -3198,17 +3360,15 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } if (sum_all > 0) { - printf("%s: hist: ", __func__); + LLAMA_LOG_INFO("%s: hist: ", __func__); for (size_t i = 0; i < hist_all.size(); i++) { - printf("%5.3f ", hist_all[i] / float(sum_all)); + LLAMA_LOG_INFO("%5.3f ", hist_all[i] / float(sum_all)); } - printf("\n"); + LLAMA_LOG_INFO("\n"); } } } - - // // interface implementation // @@ -3222,12 +3382,12 @@ struct llama_model * llama_load_model_from_file( ggml_type memory_type = params.f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32; - if (!llama_model_load(path_model, *model, model->vocab, params.n_ctx, params.n_batch, params.n_gqa, params.rms_norm_eps, params.n_gpu_layers, - params.main_gpu, params.tensor_split, params.rope_freq_base, params.rope_freq_scale,params.low_vram, - memory_type, params.use_mmap, params.use_mlock, params.vocab_only, params.progress_callback, - params.progress_callback_user_data)) { + if (!llama_model_load(path_model, *model, model->vocab, params.n_ctx, params.n_batch, params.n_gpu_layers, + params.main_gpu, params.tensor_split, params.mul_mat_q, params.rope_freq_base, params.rope_freq_scale, + params.low_vram, memory_type, params.use_mmap, params.use_mlock, params.vocab_only, + params.progress_callback, params.progress_callback_user_data)) { + LLAMA_LOG_ERROR("%s: failed to load model\n", __func__); delete model; - fprintf(stderr, "%s: failed to load model\n", __func__); return nullptr; } @@ -3260,10 +3420,9 @@ struct llama_context * llama_new_context_with_model( unsigned percentage = (unsigned) (100 * progress); while (percentage > *cur_percentage_p) { *cur_percentage_p = percentage; - fprintf(stderr, "."); - fflush(stderr); + LLAMA_LOG_INFO("."); if (percentage >= 100) { - fprintf(stderr, "\n"); + LLAMA_LOG_INFO("\n"); } } }; @@ -3277,14 +3436,14 @@ struct llama_context * llama_new_context_with_model( // reserve memory for context buffers if (!params.vocab_only) { if (!kv_cache_init(ctx->model.hparams, ctx->kv_self, memory_type, ctx->model.hparams.n_ctx, params.n_gpu_layers)) { - fprintf(stderr, "%s: kv_cache_init() failed for self-attention cache\n", __func__); + LLAMA_LOG_ERROR("%s: kv_cache_init() failed for self-attention cache\n", __func__); llama_free(ctx); return nullptr; } { const size_t memory_size = ggml_nbytes(ctx->kv_self.k) + ggml_nbytes(ctx->kv_self.v); - fprintf(stderr, "%s: kv self size = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0); + LLAMA_LOG_INFO("%s: kv self size = %7.2f MB\n", __func__, memory_size / 1024.0 / 1024.0); } const auto & hparams = ctx->model.hparams; @@ -3300,10 +3459,47 @@ struct llama_context * llama_new_context_with_model( ctx->embedding.resize(hparams.n_embd); } +#ifdef LLAMA_USE_ALLOCATOR + { + static const size_t tensor_alignment = 32; + // the compute buffer is used to store the tensor and graph structs, while the allocator buffer is used for the tensor data + ctx->buf_compute.resize(ggml_tensor_overhead()*GGML_MAX_NODES + ggml_graph_overhead()); + + // create measure allocator + ctx->alloc = ggml_allocr_new_measure(tensor_alignment); + + // build worst-case graph + int n_tokens = std::min((int)hparams.n_ctx, params.n_batch); + int n_past = hparams.n_ctx - n_tokens; + llama_token token = llama_token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph + ggml_cgraph * gf = llama_build_graph(*ctx, &token, NULL, n_tokens, n_past); + + // measure memory requirements for the graph + size_t alloc_size = ggml_allocr_alloc_graph(ctx->alloc, gf) + tensor_alignment; + + LLAMA_LOG_INFO("%s: compute buffer total size = %7.2f MB\n", __func__, (ctx->buf_compute.size + alloc_size) / 1024.0 / 1024.0); + + // debug - for comparison with scratch buffer + //size_t prev_req = + // MEM_REQ_SCRATCH0(hparams.n_ctx).at(ctx->model.type) + + // MEM_REQ_SCRATCH1().at(ctx->model.type) + + // MEM_REQ_EVAL().at(ctx->model.type); + //LLAMA_LOG_INFO("%s: (debug) equivalent with scratch buffer = %7.2f MB\n", __func__, prev_req / 1024.0 / 1024.0); + + // recreate allocator with exact memory requirements + ggml_allocr_free(ctx->alloc); + + ctx->buf_alloc.resize(alloc_size); + ctx->alloc = ggml_allocr_new(ctx->buf_alloc.data, ctx->buf_alloc.size, tensor_alignment); + } +#else ctx->buf_compute.resize(MEM_REQ_EVAL().at(ctx->model.type) + ggml_graph_overhead()); +#endif +#ifdef LLAMA_USE_SCRATCH ctx->buf_scratch[0].resize(MEM_REQ_SCRATCH0(hparams.n_ctx).at(ctx->model.type)); ctx->buf_scratch[1].resize(MEM_REQ_SCRATCH1().at(ctx->model.type)); +#endif } #ifdef GGML_USE_METAL @@ -3324,22 +3520,22 @@ struct llama_context * llama_new_context_with_model( const size_t max_size = ggml_get_max_tensor_size(ctx->model.ctx); - fprintf(stderr, "%s: max tensor size = %8.2f MB\n", __func__, max_size/1024.0/1024.0); + LLAMA_LOG_INFO("%s: max tensor size = %8.2f MB\n", __func__, max_size/1024.0/1024.0); -#define LLAMA_METAL_CHECK_BUF(result) \ - if (!(result)) { \ - fprintf(stderr, "%s: failed to add buffer\n", __func__); \ - llama_free(ctx); \ - return NULL; \ +#define LLAMA_METAL_CHECK_BUF(result) \ + if (!(result)) { \ + LLAMA_LOG_ERROR("%s: failed to add buffer\n", __func__); \ + llama_free(ctx); \ + return NULL; \ } LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "data", data_ptr, data_size, max_size)); - LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "eval", ctx->buf_compute.addr, ctx->buf_compute.size, 0)); - LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "kv", ctx->kv_self.buf.addr, ctx->kv_self.buf.size, 0)); + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "eval", ctx->buf_compute.data, ctx->buf_compute.size, 0)); + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "kv", ctx->kv_self.buf.data, ctx->kv_self.buf.size, 0)); - LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr0", ctx->buf_scratch[0].addr, ctx->buf_scratch[0].size, 0)); - LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr1", ctx->buf_scratch[1].addr, ctx->buf_scratch[1].size, 0)); + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr0", ctx->buf_scratch[0].data, ctx->buf_scratch[0].size, 0)); + LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "scr1", ctx->buf_scratch[1].data, ctx->buf_scratch[1].size, 0)); #undef LLAMA_METAL_CHECK_BUF } #endif @@ -3373,9 +3569,6 @@ struct llama_context * llama_init_from_file( } void llama_free(struct llama_context * ctx) { - if (ctx->model_owner) { - delete &ctx->model; - } delete ctx; } @@ -3387,19 +3580,19 @@ int llama_model_quantize( llama_model_quantize_internal(fname_inp, fname_out, params); return 0; } catch (const std::exception & err) { - fprintf(stderr, "%s: failed to quantize: %s\n", __func__, err.what()); + LLAMA_LOG_ERROR("%s: failed to quantize: %s\n", __func__, err.what()); return 1; } } int llama_apply_lora_from_file_internal(const struct llama_model & model, const char * path_lora, const char * path_base_model, int n_threads) { - fprintf(stderr, "%s: applying lora adapter from '%s' - please wait ...\n", __func__, path_lora); + LLAMA_LOG_INFO("%s: applying lora adapter from '%s' - please wait ...\n", __func__, path_lora); const int64_t t_start_lora_us = ggml_time_us(); auto fin = std::ifstream(path_lora, std::ios::binary); if (!fin) { - fprintf(stderr, "%s: failed to open '%s'\n", __func__, path_lora); + LLAMA_LOG_ERROR("%s: failed to open '%s'\n", __func__, path_lora); return 1; } @@ -3411,7 +3604,7 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const fin.read((char *) &format_version, sizeof(format_version)); if (format_version != 1) { - fprintf(stderr, "%s: unsupported file version\n", __func__ ); + LLAMA_LOG_ERROR("%s: unsupported file version\n", __func__ ); return 1; } } @@ -3422,8 +3615,7 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const fin.read((char *) &lora_alpha, sizeof(lora_alpha)); float scaling = (float)lora_alpha / (float)lora_r; - fprintf(stderr, "%s: r = %d, alpha = %d, scaling = %.2f\n", __func__, lora_r, lora_alpha, scaling); - + LLAMA_LOG_INFO("%s: r = %d, alpha = %d, scaling = %.2f\n", __func__, lora_r, lora_alpha, scaling); // create a temporary ggml context to store the lora tensors // todo: calculate size from biggest possible tensor @@ -3442,13 +3634,12 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const model_tensors.insert(kv); } - // load base model std::unique_ptr model_loader; ggml_context * base_ctx = NULL; - gguf_buffer base_buf; + std::vector base_buf; if (path_base_model) { - fprintf(stderr, "%s: loading base model from '%s'\n", __func__, path_base_model); + LLAMA_LOG_INFO("%s: loading base model from '%s'\n", __func__, path_base_model); model_loader.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true)); size_t ctx_size; @@ -3457,8 +3648,8 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const base_buf.resize(ctx_size); ggml_init_params base_params; - base_params.mem_size = base_buf.size; - base_params.mem_buffer = base_buf.addr; + base_params.mem_size = base_buf.size(); + base_params.mem_buffer = base_buf.data(); base_params.no_alloc = model_loader->use_mmap; base_ctx = ggml_init(base_params); @@ -3505,17 +3696,17 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const const std::string lora_suffix = ".lora"; size_t pos = name.rfind(lora_suffix); if (pos == std::string::npos) { - fprintf(stderr, "%s: error: '%s' is not a lora tensor\n", __func__, name.c_str()); + LLAMA_LOG_ERROR("%s: error: '%s' is not a lora tensor\n", __func__, name.c_str()); return 1; } std::string lora_type = name.substr(pos + lora_suffix.length()); std::string base_name = name; base_name.erase(pos); - // fprintf(stderr, "%s: %s => %s (lora type %s) ", __func__, name.c_str(),base_name.c_str(), lora_type.c_str()); + // LLAMA_LOG_INFO("%s: %s => %s (lora type %s) \n", __func__, name.c_str(),base_name.c_str(), lora_type.c_str()); if (model_tensors.find(base_name) == model_tensors.end()) { - fprintf(stderr, "%s: unknown tensor '%s' in lora adapter\n", __func__, name.data()); + LLAMA_LOG_ERROR("%s: unknown tensor '%s' in lora adapter\n", __func__, name.data()); return 1; } @@ -3526,7 +3717,7 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const case 1: wtype = GGML_TYPE_F16; break; default: { - fprintf(stderr, "%s: invalid tensor data type '%d'\n", + LLAMA_LOG_ERROR("%s: invalid tensor data type '%d'\n", __func__, ftype); return false; } @@ -3536,7 +3727,7 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const lora_tensor = ggml_new_tensor_2d(lora_ctx, wtype, ne[0], ne[1]); } else { - fprintf(stderr, "%s: unsupported tensor dimension %d\n", __func__, n_dims); + LLAMA_LOG_ERROR("%s: unsupported tensor dimension %d\n", __func__, n_dims); return 1; } ggml_set_name(lora_tensor, "lora_tensor"); @@ -3574,7 +3765,7 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const if (model_loader) { // load from base model if (model_loader->tensors_map.name_to_idx.find(base_name) == model_loader->tensors_map.name_to_idx.end()) { - fprintf(stderr, "%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str()); + LLAMA_LOG_ERROR("%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str()); return 1; } size_t idx = model_loader->tensors_map.name_to_idx[base_name]; @@ -3590,8 +3781,8 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const if (ggml_is_quantized(base_t->type)) { if (!warned) { - fprintf(stderr, "%s: warning: using a lora adapter with a quantized model may result in poor quality, " - "use a f16 or f32 base model with --lora-base\n", __func__); + LLAMA_LOG_WARN("%s: warning: using a lora adapter with a quantized model may result in poor quality, " + "use a f16 or f32 base model with --lora-base\n", __func__); warned = true; } } @@ -3605,8 +3796,8 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const ggml_set_name(loraB, "loraB"); if (base_t->ne[0] != loraA->ne[1] || base_t->ne[1] != loraB->ne[1]) { - fprintf(stderr, "%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");" - " are you sure that this adapter is for this model?\n", __func__, base_t->ne[0], loraA->ne[1]); + LLAMA_LOG_ERROR("%s: incompatible tensor dimensions (%" PRId64 " and %" PRId64 ");" + " are you sure that this adapter is for this model?\n", __func__, base_t->ne[0], loraA->ne[1]); return 1; } @@ -3651,7 +3842,7 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const n_tensors++; if (n_tensors % 4 == 0) { - fprintf(stderr, "."); + LLAMA_LOG_INFO("."); } } } @@ -3663,7 +3854,7 @@ int llama_apply_lora_from_file_internal(const struct llama_model & model, const } const int64_t t_lora_us = ggml_time_us() - t_start_lora_us; - fprintf(stderr, " done (%.2f ms)\n", t_lora_us / 1000.0); + LLAMA_LOG_INFO(" done (%.2f ms)\n", t_lora_us / 1000.0); return 0; } @@ -3672,7 +3863,7 @@ int llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lor try { return llama_apply_lora_from_file_internal(ctx->model, path_lora, path_base_model, n_threads); } catch (const std::exception & err) { - fprintf(stderr, "%s: failed to apply lora adapter: %s\n", __func__, err.what()); + LLAMA_LOG_ERROR("%s: failed to apply lora adapter: %s\n", __func__, err.what()); return 1; } } @@ -3681,7 +3872,7 @@ int llama_model_apply_lora_from_file(const struct llama_model * model, const cha try { return llama_apply_lora_from_file_internal(*model, path_lora, path_base_model, n_threads); } catch (const std::exception & err) { - fprintf(stderr, "%s: failed to apply lora adapter: %s\n", __func__, err.what()); + LLAMA_LOG_ERROR("%s: failed to apply lora adapter: %s\n", __func__, err.what()); return 1; } } @@ -3730,10 +3921,60 @@ size_t llama_get_state_size(const struct llama_context * ctx) { return s_total; } -// Copies the state to the specified destination address -size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { - uint8_t * out = dst; +// llama_context_data +struct llama_data_context { + virtual void write(const void * src, size_t size) = 0; + virtual size_t get_size_written() = 0; + virtual ~llama_data_context() = default; +}; + +struct llama_data_buffer_context : llama_data_context { + uint8_t * ptr; + size_t size_written = 0; + llama_data_buffer_context(uint8_t * p) : ptr(p) {} + + void write(const void * src, size_t size) override { + memcpy(ptr, src, size); + ptr += size; + size_written += size; + } + + size_t get_size_written() override { + return size_written; + } +}; + +struct llama_data_file_context : llama_data_context { + FILE * file; + size_t size_written = 0; + + llama_data_file_context(FILE * f) : file(f) {} + + void write(const void * src, size_t size) override { + fwrite(src, size, 1, file); + size_written += size; + } + + size_t get_size_written() override { + return size_written; + } +}; + +/** copy state data into either a buffer or file depending on the passed in context + * + * file context: + * llama_file file("/path", "wb"); + * llama_data_file_context data_ctx(&file); + * llama_copy_state_data(ctx, &data_ctx); + * + * buffer context: + * std::vector buf(max_size, 0); + * llama_data_buffer_context data_ctx(&buf.data()); + * llama_copy_state_data(ctx, &data_ctx); + * +*/ +void llama_copy_state_data_internal(struct llama_context * ctx, llama_data_context * data_ctx) { // copy rng { std::stringstream rng_ss; @@ -3745,8 +3986,8 @@ size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { memset(&rng_buf[0], 0, LLAMA_MAX_RNG_STATE); memcpy(&rng_buf[0], rng_ss.str().data(), rng_ss.str().size()); - memcpy(out, &rng_size, sizeof(rng_size)); out += sizeof(rng_size); - memcpy(out, &rng_buf[0], LLAMA_MAX_RNG_STATE); out += LLAMA_MAX_RNG_STATE; + data_ctx->write(&rng_size, sizeof(rng_size)); + data_ctx->write(&rng_buf[0], LLAMA_MAX_RNG_STATE); } // copy logits @@ -3754,25 +3995,29 @@ size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { const size_t logits_cap = ctx->logits.capacity(); const size_t logits_size = ctx->logits.size(); - memcpy(out, &logits_cap, sizeof(logits_cap)); out += sizeof(logits_cap); - memcpy(out, &logits_size, sizeof(logits_size)); out += sizeof(logits_size); + data_ctx->write(&logits_cap, sizeof(logits_cap)); + data_ctx->write(&logits_size, sizeof(logits_size)); if (logits_size) { - memcpy(out, ctx->logits.data(), logits_size * sizeof(float)); + data_ctx->write(ctx->logits.data(), logits_size * sizeof(float)); } - out += logits_cap * sizeof(float); + // If there is a gap between the size and the capacity, write padding + size_t padding_size = (logits_cap - logits_size) * sizeof(float); + if (padding_size > 0) { + std::vector padding(padding_size, 0); // Create a buffer filled with zeros + data_ctx->write(padding.data(), padding_size); + } } // copy embeddings { const size_t embedding_size = ctx->embedding.size(); - memcpy(out, &embedding_size, sizeof(embedding_size)); out += sizeof(embedding_size); + data_ctx->write(&embedding_size, sizeof(embedding_size)); if (embedding_size) { - memcpy(out, ctx->embedding.data(), embedding_size * sizeof(float)); - out += embedding_size * sizeof(float); + data_ctx->write(ctx->embedding.data(), embedding_size * sizeof(float)); } } @@ -3781,14 +4026,14 @@ size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { const auto & kv_self = ctx->kv_self; const auto & hparams = ctx->model.hparams; const int n_layer = hparams.n_layer; - const int n_embd = hparams.n_embd; + const int n_embd = hparams.n_embd_gqa(); const int n_ctx = hparams.n_ctx; const size_t kv_size = kv_self.buf.size; const int kv_ntok = llama_get_kv_cache_token_count(ctx); - memcpy(out, &kv_size, sizeof(kv_size)); out += sizeof(kv_size); - memcpy(out, &kv_ntok, sizeof(kv_ntok)); out += sizeof(kv_ntok); + data_ctx->write(&kv_size, sizeof(kv_size)); + data_ctx->write(&kv_ntok, sizeof(kv_ntok)); if (kv_size) { const size_t elt_size = ggml_element_size(kv_self.k); @@ -3797,12 +4042,12 @@ size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { ggml_cgraph gf{}; ggml_tensor * kout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.k->type, n_embd, kv_ntok, n_layer); - kout3d->data = out; - out += ggml_nbytes(kout3d); + std::vector kout3d_data(ggml_nbytes(kout3d), 0); + kout3d->data = kout3d_data.data(); ggml_tensor * vout3d = ggml_new_tensor_3d(cpy_ctx, kv_self.v->type, kv_ntok, n_embd, n_layer); - vout3d->data = out; - out += ggml_nbytes(vout3d); + std::vector vout3d_data(ggml_nbytes(vout3d), 0); + vout3d->data = vout3d_data.data(); ggml_tensor * k3d = ggml_view_3d(cpy_ctx, kv_self.k, n_embd, kv_ntok, n_layer, @@ -3817,15 +4062,20 @@ size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { ggml_graph_compute_helper(ctx->work_buffer, &gf, /*n_threads*/ 1); ggml_free(cpy_ctx); + + // our data is now in the kout3d_data and vout3d_data buffers + // write them to file + data_ctx->write(kout3d_data.data(), kout3d_data.size()); + data_ctx->write(vout3d_data.data(), vout3d_data.size()); } } +} - const size_t written = out - dst; - const size_t max_size = llama_get_state_size(ctx); - - GGML_ASSERT(written <= max_size); +size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { + llama_data_buffer_context data_ctx(dst); + llama_copy_state_data_internal(ctx, &data_ctx); - return written; + return data_ctx.get_size_written(); } // Sets the state reading from the specified source address @@ -3884,7 +4134,7 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { const auto & kv_self = ctx->kv_self; const auto & hparams = ctx->model.hparams; const int n_layer = hparams.n_layer; - const int n_embd = hparams.n_embd; + const int n_embd = hparams.n_embd_gqa(); const int n_ctx = hparams.n_ctx; size_t kv_size; @@ -3952,7 +4202,7 @@ bool llama_load_session_file(struct llama_context * ctx, const char * path_sessi try { return llama_load_session_file_internal(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out); } catch (const std::exception & err) { - fprintf(stderr, "error loading session file: %s\n", err.what()); + LLAMA_LOG_ERROR("error loading session file: %s\n", err.what()); return false; } } @@ -3975,7 +4225,7 @@ int llama_eval( int n_past, int n_threads) { if (!llama_eval_internal(*ctx, tokens, nullptr, n_tokens, n_past, n_threads, nullptr)) { - fprintf(stderr, "%s: failed to eval\n", __func__); + LLAMA_LOG_ERROR("%s: failed to eval\n", __func__); return 1; } @@ -3997,7 +4247,7 @@ int llama_eval_embd( int n_past, int n_threads) { if (!llama_eval_internal(*ctx, nullptr, embd, n_tokens, n_past, n_threads, nullptr)) { - fprintf(stderr, "%s: failed to eval\n", __func__); + LLAMA_LOG_ERROR("%s: failed to eval\n", __func__); return 1; } @@ -4018,7 +4268,7 @@ int llama_eval_export(struct llama_context * ctx, const char * fname) { const std::vector tmp(n_batch, llama_token_bos()); if (!llama_eval_internal(*ctx, tmp.data(), nullptr, tmp.size(), n_ctx, 1, fname)) { - fprintf(stderr, "%s: failed to eval\n", __func__); + LLAMA_LOG_ERROR("%s: failed to eval\n", __func__); return 1; } @@ -4034,7 +4284,7 @@ int llama_tokenize_with_model( auto res = llama_tokenize(model->vocab, text, add_bos); if (n_max_tokens < (int) res.size()) { - fprintf(stderr, "%s: too many tokens\n", __func__); + LLAMA_LOG_ERROR("%s: too many tokens\n", __func__); return -((int) res.size()); } @@ -4151,15 +4401,15 @@ struct llama_timings llama_get_timings(struct llama_context * ctx) { void llama_print_timings(struct llama_context * ctx) { const llama_timings timings = llama_get_timings(ctx); - fprintf(stderr, "\n"); - fprintf(stderr, "%s: load time = %8.2f ms\n", __func__, timings.t_load_ms); - fprintf(stderr, "%s: sample time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", + LLAMA_LOG_INFO("\n"); + LLAMA_LOG_INFO("%s: load time = %8.2f ms\n", __func__, timings.t_load_ms); + LLAMA_LOG_INFO("%s: sample time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", __func__, timings.t_sample_ms, timings.n_sample, timings.t_sample_ms / timings.n_sample, 1e3 / timings.t_sample_ms * timings.n_sample); - fprintf(stderr, "%s: prompt eval time = %8.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", + LLAMA_LOG_INFO("%s: prompt eval time = %8.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", __func__, timings.t_p_eval_ms, timings.n_p_eval, timings.t_p_eval_ms / timings.n_p_eval, 1e3 / timings.t_p_eval_ms * timings.n_p_eval); - fprintf(stderr, "%s: eval time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", + LLAMA_LOG_INFO("%s: eval time = %8.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", __func__, timings.t_eval_ms, timings.n_eval, timings.t_eval_ms / timings.n_eval, 1e3 / timings.t_eval_ms * timings.n_eval); - fprintf(stderr, "%s: total time = %8.2f ms\n", __func__, (timings.t_end_ms - timings.t_start_ms)); + LLAMA_LOG_INFO("%s: total time = %8.2f ms\n", __func__, (timings.t_end_ms - timings.t_start_ms)); } void llama_reset_timings(struct llama_context * ctx) { @@ -4195,3 +4445,44 @@ const char * llama_print_system_info(void) { const std::vector>& llama_internal_get_tensor_map(struct llama_context * ctx) { return ctx->model.tensors_by_name; } + + +void llama_log_set(llama_log_callback log_callback, void * user_data) { + g_state.log_callback = log_callback ? log_callback : llama_log_callback_default; + g_state.log_callback_user_data = user_data; +} + +#if defined(_MSC_VER) && !defined(vsnprintf) +#define vsnprintf _vsnprintf +#endif + +static void llama_log_internal_v(llama_log_level level, const char * format, va_list args) { + va_list args_copy; + va_copy(args_copy, args); + char buffer[128]; + int len = vsnprintf(buffer, 128, format, args); + if (len < 128) { + g_state.log_callback(level, buffer, g_state.log_callback_user_data); + } else { + char* buffer2 = new char[len+1]; + vsnprintf(buffer2, len+1, format, args_copy); + buffer2[len] = 0; + g_state.log_callback(level, buffer2, g_state.log_callback_user_data); + delete[] buffer2; + } + va_end(args_copy); +} + +static void llama_log_internal(llama_log_level level, const char * format, ...) { + va_list args; + va_start(args, format); + llama_log_internal_v(level, format, args); + va_end(args); +} + +static void llama_log_callback_default(llama_log_level level, const char * text, void * user_data) { + (void) level; + (void) user_data; + fputs(text, stderr); + fflush(stderr); +} diff --git a/gguf-llama.h b/gguf-llama.h index 540167bd196ba..a8ed69d918048 100644 --- a/gguf-llama.h +++ b/gguf-llama.h @@ -41,10 +41,6 @@ #define LLAMA_SUPPORTS_GPU_OFFLOAD #endif -#ifndef LLAMA_DEFAULT_RMS_EPS -#define LLAMA_DEFAULT_RMS_EPS 5e-6f -#endif - #ifdef __cplusplus extern "C" { #endif @@ -74,12 +70,23 @@ extern "C" { typedef void (*llama_progress_callback)(float progress, void *ctx); - struct llama_context_params { + enum llama_log_level { + LLAMA_LOG_LEVEL_ERROR = 2, + LLAMA_LOG_LEVEL_WARN = 3, + LLAMA_LOG_LEVEL_INFO = 4 + }; + + // Signature for logging events + // Note that text includes the new line character at the end for most events. + // If your logging mechanism cannot handle that, check if the last character is '\n' and strip it + // if it exists. + // It might not exist for progress report where '.' is output repeatedly. + typedef void (*llama_log_callback)(enum llama_log_level level, const char * text, void * user_data); + + struct llama_context_params { uint32_t seed; // RNG seed, -1 for random int32_t n_ctx; // text context int32_t n_batch; // prompt processing batch size - int32_t n_gqa; // grouped-query attention (TEMP - will be moved to model hparams) - float rms_norm_eps; // rms norm epsilon (TEMP - will be moved to model hparams) int32_t n_gpu_layers; // number of layers to store in VRAM int32_t main_gpu; // the GPU that is used for scratch and small tensors @@ -96,6 +103,7 @@ extern "C" { // Keep the booleans together to avoid misalignment during copy-by-value. bool low_vram; // if true, reduce VRAM usage at the cost of performance + bool mul_mat_q; // if true, use experimental mul_mat_q kernels bool f16_kv; // use fp16 for KV cache bool logits_all; // the llama_eval() call computes all logits, not just the last one bool vocab_only; // only load the vocabulary, no weights @@ -129,7 +137,7 @@ extern "C" { // model quantization parameters typedef struct llama_model_quantize_params { int nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() - enum llama_ftype ftype; // quantize to this llama_ftype + enum llama_ftype ftype; // quantize to this llama_ftype bool allow_requantize; // allow quantizing non-f32/f16 tensors bool quantize_output_tensor; // quantize output.weight } llama_model_quantize_params; @@ -182,6 +190,10 @@ extern "C" { int32_t n_eval; }; + // Set callback for all future logging events. + // If this is not called, or NULL is supplied, everything is output on stderr. + LLAMA_API void llama_log_set(llama_log_callback log_callback, void * user_data); + LLAMA_API int llama_max_devices(); LLAMA_API struct llama_context_params llama_context_default_params(); diff --git a/gguf-util.h b/gguf-util.h index 774ae57eef3db..d8557d94f114d 100644 --- a/gguf-util.h +++ b/gguf-util.h @@ -64,13 +64,6 @@ static std::string format(const char * fmt, ...) { return std::string(buf.data(), size); } -template -static std::string to_string(const T & val) { - std::stringstream ss; - ss << val; - return ss.str(); -} - // TODO: can we merge this one and gguf_context? struct gguf_file { // use FILE * so we don't have to re-open the file to mmap @@ -474,94 +467,4 @@ struct gguf_mlock { #endif }; -// Replacement for std::vector that doesn't require zero-initialization. -struct gguf_buffer { - uint8_t * addr = NULL; - size_t size = 0; - - gguf_buffer() = default; - - void resize(size_t len) { -#ifdef GGML_USE_METAL - free(addr); - int result = posix_memalign((void **) &addr, getpagesize(), len); - if (result == 0) { - memset(addr, 0, len); - } - else { - addr = NULL; - } -#else - delete[] addr; - addr = new uint8_t[len]; -#endif - size = len; - } - - ~gguf_buffer() { -#ifdef GGML_USE_METAL - free(addr); -#else - delete[] addr; -#endif - addr = NULL; - } - - // disable copy and move - gguf_buffer(const gguf_buffer&) = delete; - gguf_buffer(gguf_buffer&&) = delete; - gguf_buffer& operator=(const gguf_buffer&) = delete; - gguf_buffer& operator=(gguf_buffer&&) = delete; -}; - -#ifdef GGML_USE_CUBLAS -#include "ggml-cuda.h" -struct gguf_ctx_buffer { - uint8_t * addr = NULL; - bool is_cuda; - size_t size = 0; - - gguf_ctx_buffer() = default; - - void resize(size_t size) { - free(); - - addr = (uint8_t *) ggml_cuda_host_malloc(size); - if (addr) { - is_cuda = true; - } - else { - // fall back to pageable memory - addr = new uint8_t[size]; - is_cuda = false; - } - this->size = size; - } - - void free() { - if (addr) { - if (is_cuda) { - ggml_cuda_host_free(addr); - } - else { - delete[] addr; - } - } - addr = NULL; - } - - ~gguf_ctx_buffer() { - free(); - } - - // disable copy and move - gguf_ctx_buffer(const gguf_ctx_buffer&) = delete; - gguf_ctx_buffer(gguf_ctx_buffer&&) = delete; - gguf_ctx_buffer& operator=(const gguf_ctx_buffer&) = delete; - gguf_ctx_buffer& operator=(gguf_ctx_buffer&&) = delete; -}; -#else -typedef gguf_buffer gguf_ctx_buffer; -#endif - #endif From 6f1485488096faa2ddc1aa4f10e6f1879622d61e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 14 Aug 2023 16:39:02 +0300 Subject: [PATCH 145/242] gitignore : add gptneox-main --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 13f728ed0bbf6..107caa228b89e 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ models-mnt /embd-input-test /gguf /gguf-llama-simple +/gptneox-main /libllama.so build-info.h arm_neon.h @@ -67,7 +68,6 @@ perf-*.txt examples/jeopardy/results.txt - pyproject.toml poetry.lock poetry.toml From ec1b100720a66bd0bdcc6fa65d49145c07c25eec Mon Sep 17 00:00:00 2001 From: goerch Date: Mon, 14 Aug 2023 18:30:28 +0200 Subject: [PATCH 146/242] llama : tokenizer fixes (#2549) * Merge tokenizer fixes into the gguf branch. * Add test vocabularies --- convert.py | 103 +++--- examples/common.cpp | 11 - examples/common.h | 7 +- examples/embedding/embedding.cpp | 2 +- examples/main/main.cpp | 12 +- examples/quantize-stats/quantize-stats.cpp | 1 + examples/save-load-state/save-load-state.cpp | 9 +- examples/simple/simple.cpp | 4 +- .../train-text-from-scratch.cpp | 20 +- llama.cpp | 324 ++++++++++++++++-- llama.h | 60 +++- models/ggml-vocab-aquila.bin | Bin 0 -> 1963875 bytes models/ggml-vocab-llama.bin | Bin 0 -> 466955 bytes models/ggml-vocab.bin | Bin 432610 -> 0 bytes tests/CMakeLists.txt | 37 +- tests/test-tokenizer-0.cpp | 45 ++- tests/test-tokenizer-1.cpp | 122 +++++++ 17 files changed, 611 insertions(+), 146 deletions(-) create mode 100644 models/ggml-vocab-aquila.bin create mode 100644 models/ggml-vocab-llama.bin delete mode 100644 models/ggml-vocab.bin create mode 100644 tests/test-tokenizer-1.cpp diff --git a/convert.py b/convert.py index f3bf1798089cc..d9d0c0b38215e 100644 --- a/convert.py +++ b/convert.py @@ -238,22 +238,58 @@ def load(model_plus: 'ModelPlus') -> 'Params': return params -class SentencePieceVocab: - def __init__(self, fname_tokenizer: Path, fname_added_tokens: Optional[Path], vocabtype: Optional[str]) -> None: - self.vocabtype = vocabtype - if self.vocabtype == "bpe": - self.sentencepiece_tokenizer = json.loads(open(str(fname_tokenizer)).read()) - else: - self.sentencepiece_tokenizer = SentencePieceProcessor(str(fname_tokenizer)) +class BpeVocab: + def __init__(self, fname_tokenizer: Path, fname_added_tokens: Optional[Path]) -> None: + self.bpe_tokenizer = json.loads(open(str(fname_tokenizer), encoding="utf-8").read()) added_tokens: Dict[str, int] if fname_added_tokens is not None: - added_tokens = json.load(open(fname_added_tokens)) + added_tokens = json.load(open(fname_added_tokens, encoding="utf-8")) else: added_tokens = {} - if self.vocabtype == "bpe": - vocab_size: int = len(self.sentencepiece_tokenizer) + vocab_size: int = len(self.bpe_tokenizer) + expected_ids = list(range(vocab_size, vocab_size + len(added_tokens))) + actual_ids = sorted(added_tokens.values()) + if expected_ids != actual_ids: + raise Exception(f"Expected added token IDs to be sequential and start at {len(added_tokens)}; got {actual_ids}") + items = sorted(added_tokens.items(), key=lambda text_idx: text_idx[1]) + self.added_tokens_list = [text for (text, idx) in items] + self.vocab_size_base: int = vocab_size + self.vocab_size: int = self.vocab_size_base + len(self.added_tokens_list) + self.fname_tokenizer = fname_tokenizer + self.fname_added_tokens = fname_added_tokens + + def bpe_tokens(self) -> Iterable[Tuple[bytes, float]]: + tokenizer = self.bpe_tokenizer + from transformers.models.gpt2 import tokenization_gpt2 + byte_encoder = tokenization_gpt2.bytes_to_unicode() + byte_decoder = {v: k for k, v in byte_encoder.items()} + for i, item in enumerate(tokenizer): + text: bytes = item.encode("utf-8") + score: float = -i + yield text, score + + def added_tokens(self) -> Iterable[Tuple[bytes, float]]: + for text in self.added_tokens_list: + score = -1000.0 + yield text.encode("utf-8"), score + + def all_tokens(self) -> Iterable[Tuple[bytes, float]]: + yield from self.bpe_tokens() + yield from self.added_tokens() + + def __repr__(self) -> str: + return f"BpeVocab with {self.vocab_size_base} base tokens and {len(self.added_tokens_list)} added tokens>" + + +class SentencePieceVocab: + def __init__(self, fname_tokenizer: Path, fname_added_tokens: Optional[Path]) -> None: + self.sentencepiece_tokenizer = SentencePieceProcessor(str(fname_tokenizer)) + added_tokens: Dict[str, int] + if fname_added_tokens is not None: + added_tokens = json.load(open(fname_added_tokens, encoding="utf-8")) else: - vocab_size: int = self.sentencepiece_tokenizer.vocab_size() + added_tokens = {} + vocab_size: int = self.sentencepiece_tokenizer.vocab_size() expected_ids = list(range(vocab_size, vocab_size + len(added_tokens))) actual_ids = sorted(added_tokens.values()) if expected_ids != actual_ids: @@ -267,32 +303,11 @@ def __init__(self, fname_tokenizer: Path, fname_added_tokens: Optional[Path], vo def sentencepiece_tokens(self) -> Iterable[Tuple[bytes, float]]: tokenizer = self.sentencepiece_tokenizer - if self.vocabtype == "bpe": - from transformers.models.gpt2 import tokenization_gpt2 - byte_encoder = tokenization_gpt2.bytes_to_unicode() - byte_decoder = {v: k for k, v in byte_encoder.items()} - for i, item in enumerate(tokenizer): - text: bytes - text = b''.join([x.to_bytes(1, byteorder='big') for x in [byte_decoder[y] for y in item]]) - score: float = -i + for i in range(tokenizer.vocab_size()): + piece = tokenizer.id_to_piece(i) + text: bytes = piece.encode("utf-8") + score: float = tokenizer.get_score(i) yield text, score - else: - for i in range(tokenizer.vocab_size()): - text: bytes - if tokenizer.is_unknown(i): - text = " \u2047 ".encode("utf-8") - elif tokenizer.is_control(i): - text = b"" - elif tokenizer.is_byte(i): - piece = tokenizer.id_to_piece(i) - if len(piece) != 6: - raise Exception(f"Invalid token: {piece}") - byte_value = int(piece[3:-1], 16) - text = struct.pack("B", byte_value) - else: - text = tokenizer.id_to_piece(i).replace("\u2581", " ").encode("utf-8") - score: float = tokenizer.get_score(i) - yield text, score def added_tokens(self) -> Iterable[Tuple[bytes, float]]: for text in self.added_tokens_list: @@ -319,7 +334,7 @@ def __repr__(self) -> str: return f"" -Vocab = Union[SentencePieceVocab, GGMLVocab] +Vocab = Union[BpeVocab, SentencePieceVocab, GGMLVocab] def permute(weights: NDArray, n_head: int, n_kv_head: Optional[int] = None) -> NDArray: @@ -1044,7 +1059,7 @@ def bounded_parallel_map(func: Callable[[In], Out], iterable: Iterable[In], conc def check_vocab_size(params: Params, vocab: Vocab) -> None: if params.n_vocab != vocab.vocab_size: # GGMLVocab comes from the same file as the model so shouldn't mismatch: - assert isinstance(vocab, SentencePieceVocab) + assert isinstance(vocab, BpeVocab) or isinstance(vocab, SentencePieceVocab) if params.n_vocab == vocab.vocab_size_base: print("Ignoring added_tokens.json since model matches vocab size without it.") vocab.added_tokens_list = [] @@ -1093,7 +1108,7 @@ def write_vocab(self, vocab: Vocab) -> None: @staticmethod def write_vocab_only(fname_out: Path, vocab: Vocab) -> None: of = OutputFile(fname_out) - params = Params(n_vocab=vocab.vocab_size, n_embd=0, n_mult=0, n_head=1, n_layer=0) + params = Params(n_vocab=vocab.vocab_size, n_embd=0, n_mult=0, n_head=1, n_layer=0, n_kv_head=None) of = OutputFile(fname_out) of.write_file_header(params, file_type=GGMLFileType.AllF32) of.write_vocab(vocab) @@ -1228,7 +1243,7 @@ def filter_and_sort_tensors(model: LazyModel) -> LazyModel: return {name: model[name] for name in TENSORS_LIST if name in model} -def load_vocab(path: Path, vocabtype: Optional[str]) -> SentencePieceVocab: +def load_vocab(path: Path, vocabtype: Optional[str]) -> Union[BpeVocab, SentencePieceVocab]: print(f"vocabtype: {vocabtype}") # Be extra-friendly and accept either a file or a directory. Also, if it's # a directory, it might be the model directory, and tokenizer.model might @@ -1250,8 +1265,12 @@ def load_vocab(path: Path, vocabtype: Optional[str]) -> SentencePieceVocab: "if it's in another directory, pass the directory as --vocab-dir") added_tokens_path = path.parent / "added_tokens.json" print(f"Loading vocab file {path}") - return SentencePieceVocab(path, added_tokens_path if added_tokens_path.exists() else None, - vocabtype) + if vocabtype == "bpe": + return BpeVocab(path, added_tokens_path if added_tokens_path.exists() else None) + elif vocabtype == "spm": + return SentencePieceVocab(path, added_tokens_path if added_tokens_path.exists() else None) + else: + raise ValueError(f"Unsupported vocabulary type {vocabtype}") def default_outfile(model_paths: List[Path], file_type: GGMLFileType) -> Path: diff --git a/examples/common.cpp b/examples/common.cpp index 9f8aab9a25681..18de8ef43d403 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -633,17 +633,6 @@ std::string gpt_random_prompt(std::mt19937 & rng) { return "The"; } -// TODO: not great allocating this every time -std::vector llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos) { - // initialize to prompt numer of chars, since n_tokens <= n_prompt_chars - std::vector res(text.size() + (int) add_bos); - const int n = llama_tokenize(ctx, text.c_str(), res.data(), res.size(), add_bos); - assert(n >= 0); - res.resize(n); - - return res; -} - struct llama_context_params llama_context_params_from_gpt_params(const gpt_params & params) { auto lparams = llama_context_default_params(); diff --git a/examples/common.h b/examples/common.h index 375bc0a3db416..0e520ed4e7775 100644 --- a/examples/common.h +++ b/examples/common.h @@ -2,6 +2,7 @@ #pragma once +#define LLAMA_API_CPP // TODO: eliminate me #include "llama.h" #include @@ -100,12 +101,6 @@ void gpt_print_usage(int argc, char ** argv, const gpt_params & params); std::string gpt_random_prompt(std::mt19937 & rng); -// -// Vocab utils -// - -std::vector llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos); - // // Model utils // diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index 5192d6df5c2f8..8788571cbf9d4 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -67,7 +67,7 @@ int main(int argc, char ** argv) { fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str()); fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size()); for (int i = 0; i < (int) embd_inp.size(); i++) { - fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], llama_token_to_str(ctx, embd_inp[i])); + fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], llama_token_to_str(ctx, embd_inp[i]).c_str()); } fprintf(stderr, "\n"); } diff --git a/examples/main/main.cpp b/examples/main/main.cpp index a632bea1cf2b9..2424435607b0f 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -191,10 +191,6 @@ int main(int argc, char ** argv) { // tokenize the prompt std::vector embd_inp; - - // Add a space in front of the first character to match OG llama tokenizer behavior - params.prompt.insert(0, 1, ' '); - if (params.interactive_first || params.instruct || !params.prompt.empty() || session_tokens.empty()) { embd_inp = ::llama_tokenize(ctx, params.prompt, true); } else { @@ -278,7 +274,7 @@ int main(int argc, char ** argv) { fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str()); fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size()); for (int i = 0; i < (int) embd_inp.size(); i++) { - fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], llama_token_to_str(ctx, embd_inp[i])); + fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], llama_token_to_str(ctx, embd_inp[i]).c_str()); } if (ctx_guidance) { @@ -286,14 +282,14 @@ int main(int argc, char ** argv) { fprintf(stderr, "%s: negative prompt: '%s'\n", __func__, params.cfg_negative_prompt.c_str()); fprintf(stderr, "%s: number of tokens in negative prompt = %zu\n", __func__, guidance_inp.size()); for (int i = 0; i < (int) guidance_inp.size(); i++) { - fprintf(stderr, "%6d -> '%s'\n", guidance_inp[i], llama_token_to_str(ctx, guidance_inp[i])); + fprintf(stderr, "%6d -> '%s'\n", guidance_inp[i], llama_token_to_str(ctx, guidance_inp[i]).c_str()); } } if (params.n_keep > 0) { fprintf(stderr, "%s: static prompt based on n_keep: '", __func__); for (int i = 0; i < params.n_keep; i++) { - fprintf(stderr, "%s", llama_token_to_str(ctx, embd_inp[i])); + fprintf(stderr, "%s", llama_token_to_str(ctx, embd_inp[i]).c_str()); } fprintf(stderr, "'\n"); } @@ -662,7 +658,7 @@ int main(int argc, char ** argv) { // display text if (input_echo) { for (auto id : embd) { - printf("%s", llama_token_to_str(ctx, id)); + printf("%s", llama_token_to_str(ctx, id).c_str()); } fflush(stdout); } diff --git a/examples/quantize-stats/quantize-stats.cpp b/examples/quantize-stats/quantize-stats.cpp index 6aa06ec8fa115..a330b20df2963 100644 --- a/examples/quantize-stats/quantize-stats.cpp +++ b/examples/quantize-stats/quantize-stats.cpp @@ -1,6 +1,7 @@ #include "ggml.h" #include "build-info.h" +#define LLAMA_API_CPP // TODO: eliminate me #define LLAMA_API_INTERNAL #include "llama.h" diff --git a/examples/save-load-state/save-load-state.cpp b/examples/save-load-state/save-load-state.cpp index 61c71c3589fdf..d5a81978e97af 100644 --- a/examples/save-load-state/save-load-state.cpp +++ b/examples/save-load-state/save-load-state.cpp @@ -45,9 +45,8 @@ int main(int argc, char ** argv) { llama_free_model(model); return 1; } - auto tokens = std::vector(params.n_ctx); - auto n_prompt_tokens = llama_tokenize(ctx, params.prompt.c_str(), tokens.data(), int(tokens.size()), true); - + auto tokens = llama_tokenize(ctx, params.prompt.c_str(), true); + auto n_prompt_tokens = tokens.size(); if (n_prompt_tokens < 1) { fprintf(stderr, "%s : failed to tokenize prompt\n", __func__); llama_free(ctx); @@ -92,7 +91,7 @@ int main(int argc, char ** argv) { auto next_token_str = llama_token_to_str(ctx, next_token); last_n_tokens_data.push_back(next_token); - printf("%s", next_token_str); + printf("%s", next_token_str.c_str()); if (llama_eval(ctx, &next_token, 1, n_past, params.n_threads)) { fprintf(stderr, "\n%s : failed to evaluate\n", __func__); llama_free(ctx); @@ -152,7 +151,7 @@ int main(int argc, char ** argv) { auto next_token_str = llama_token_to_str(ctx2, next_token); last_n_tokens_data.push_back(next_token); - printf("%s", next_token_str); + printf("%s", next_token_str.c_str()); if (llama_eval(ctx2, &next_token, 1, n_past, params.n_threads)) { fprintf(stderr, "\n%s : failed to evaluate\n", __func__); llama_free(ctx2); diff --git a/examples/simple/simple.cpp b/examples/simple/simple.cpp index 55cce10442d58..0dfcd7a448826 100644 --- a/examples/simple/simple.cpp +++ b/examples/simple/simple.cpp @@ -62,7 +62,7 @@ int main(int argc, char ** argv) { fprintf(stderr, "\n\n"); for (auto id : tokens_list) { - fprintf(stderr, "%s", llama_token_to_str(ctx, id)); + fprintf(stderr, "%s", llama_token_to_str(ctx, id).c_str()); } fflush(stderr); @@ -109,7 +109,7 @@ int main(int argc, char ** argv) { } // print the new token : - printf("%s", llama_token_to_str(ctx, new_token_id)); + printf("%s", llama_token_to_str(ctx, new_token_id).c_str()); fflush(stdout); // push this new token for next evaluation diff --git a/examples/train-text-from-scratch/train-text-from-scratch.cpp b/examples/train-text-from-scratch/train-text-from-scratch.cpp index 54dc2beed0080..d446c6ea5cfb2 100644 --- a/examples/train-text-from-scratch/train-text-from-scratch.cpp +++ b/examples/train-text-from-scratch/train-text-from-scratch.cpp @@ -1,4 +1,5 @@ #include "ggml.h" +#include "common.h" #include "llama.h" #include #include @@ -1961,7 +1962,7 @@ void print_matrix(struct ggml_tensor * probs) { void print_token(struct llama_context * ctx, llama_token token) { - printf("%s", llama_token_to_str(ctx, token)); + printf("%s", llama_token_to_str(ctx, token).c_str()); } void print_tokens(struct llama_context* ctx, struct ggml_tensor * tokens) { @@ -2188,11 +2189,10 @@ int tokenize_file(struct llama_context * lctx, const char * filename, std::vecto f.read_raw(buf.data(), f.size); buf[f.size] = '\0'; - out.resize(buf.size()); - - int n_tokens = llama_tokenize(lctx, buf.data(), out.data(), buf.size(), false); - if (n_tokens >= 0) { - out.resize(n_tokens); + int n_tokens = llama_tokenize(lctx, buf.data(), out.data(), out.size(), false); + if (n_tokens < 0) { + out.resize(-n_tokens); + llama_tokenize(lctx, buf.data(), out.data(), out.size(), false); } bool verify = false; @@ -2200,17 +2200,17 @@ int tokenize_file(struct llama_context * lctx, const char * filename, std::vecto const char * in = buf.data(); const char * end = buf.data() + buf.size(); for (int i = 0; i < (int) out.size(); ++i) { - const char * s = llama_token_to_str(lctx, out[i]); - int len = strlen(s); + std::string s = llama_token_to_str(lctx, out[i]); + int len = s.length(); if (in >= end) { printf("%s: unexpected end of original text.\n", __func__); break; } - const bool matches = (strncmp(in, s, len) == 0); + const bool matches = (strncmp(in, s.c_str(), len) == 0); if (matches) { in += len; } else { - printf("%s: mismatch: expected '%s', but got '%s'\n", __func__, std::string(in, len).c_str(), s); + printf("%s: mismatch: expected '%s', but got '%s'\n", __func__, std::string(in, len).c_str(), s.c_str()); } } } diff --git a/llama.cpp b/llama.cpp index c8ab313d9c116..5504f14834c35 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7,6 +7,7 @@ #endif #include "llama-util.h" +#define LLAMA_API_CPP // TODO: eliminate me #include "llama.h" #include "ggml.h" @@ -575,6 +576,7 @@ struct llama_file_loader { float score = 0.0f; file.read_raw(&score, sizeof(score)); + GGML_ASSERT(vocab.token_to_id.find(word) == vocab.token_to_id.end()); vocab.token_to_id[word] = i; auto & tok_score = vocab.id_to_token[i]; @@ -1060,6 +1062,11 @@ static void llama_model_load_internal( std::unique_ptr ml(new llama_model_loader(fname, use_mmap)); vocab = std::move(ml->file_loader->vocab); + + if (vocab_only) { + return; + } + model.hparams = ml->file_loader->hparams; model.n_gpu_layers = n_gpu_layers; llama_file_version file_version = ml->file_loader->file_version; @@ -1141,10 +1148,6 @@ static void llama_model_load_internal( } } - if (vocab_only) { - return; - } - auto & ctx = model.ctx; size_t ctx_size; @@ -1940,6 +1943,105 @@ static bool llama_eval_internal( // tokenizer // +static std::string llama_vocab_type(const llama_vocab& vocab) { + return vocab.token_to_id.size() == 32000 ? "spm": "bpe"; +} + +static bool llama_is_normal_token(const llama_vocab& vocab, llama_token token) { + if(llama_vocab_type(vocab) == "spm") + return token >= 259; + else if(llama_vocab_type(vocab) == "bpe") + return token >= 95; + else + return false; +} + +static bool llama_is_unknown_token(const llama_vocab& vocab, llama_token token) { + if(llama_vocab_type(vocab) == "spm") + return token == 0; + else + // TODO: improve? + return false; +} + +static bool llama_is_control_token(const llama_vocab& vocab, llama_token token) { + if(llama_vocab_type(vocab) == "spm") + return token == 1 || token == 2; + else + // TODO: improve? + return false; +} + +static bool llama_is_bos_token(const llama_vocab& vocab, llama_token token) { + if(llama_vocab_type(vocab) == "spm") + return token == 1; + else + // TODO: improve? + return false; +} + +static bool llama_is_eos_token(const llama_vocab& vocab, llama_token token) { + if(llama_vocab_type(vocab) == "spm") + return token == 2; + else + // TODO: improve? + return false; +} + +static bool llama_is_user_defined_token(const llama_vocab& vocab, llama_token token) { + // TODO: improve? + return false; +} + +static bool llama_is_unused_token(const llama_vocab& vocab, llama_token token) { + // TODO: improve? + return false; +} + +static bool llama_is_byte_token(const llama_vocab& vocab, llama_token token) { + if(llama_vocab_type(vocab) == "spm") + return 3 <= token && token < 259; + else if(llama_vocab_type(vocab) == "bpe") + return 1 <= token && token < 95; + else + return false; +} + +static uint8_t llama_byte_to_char(const llama_vocab& vocab, uint8_t byte) { + if(llama_vocab_type(vocab) == "spm") + return byte + 3; + else if(llama_vocab_type(vocab) == "bpe") + return byte + 32; + else + return false; +} + +static std::string llama_escape_whitespace(const std::string& text) { + std::string result; + bool escaping = false; + result += "\xe2\x96\x81"; + for (size_t offs = 0; offs < text.length(); ++offs) { + if (text[offs] == ' ') { + if (!escaping) { + result += "\xe2\x96\x81"; + escaping = true; + } + } + else { + escaping = false; + result += text[offs]; + } + } + return result; +} + +static std::string llama_unescape_whitespace(const std::string& word) { + if (word.length() >= 3 && word.substr(0, 3) == "\xe2\x96\x81") { + return std::string(" ") + word.substr(3); + } + return word; +} + static size_t utf8_len(char src) { const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; uint8_t highbits = static_cast(src) >> 4; @@ -1981,10 +2083,11 @@ struct llama_tokenizer { size_t offs = 0; while (offs < text.size()) { llama_sp_symbol sym; - size_t char_len = std::min(text.size() - offs, utf8_len(text[offs])); + size_t len = utf8_len(text[offs]); + GGML_ASSERT(offs + len <= text.size()); sym.text = text.c_str() + offs; - sym.n = char_len; - offs += char_len; + sym.n = len; + offs += len; sym.prev = index - 1; sym.next = offs == text.size() ? -1 : index + 1; index++; @@ -2029,23 +2132,36 @@ struct llama_tokenizer { for (int i = 0; i != -1; i = symbols_[i].next) { auto & symbol = symbols_[i]; - auto token = vocab_.token_to_id.find(std::string(symbol.text, symbol.n)); - - if (token == vocab_.token_to_id.end()) { - // output any symbols that did not form tokens as bytes. - for (int j = 0; j < (int) symbol.n; ++j) { - // NOTE: old version, before #2420 - not sure what are the implications of this - //llama_vocab::id token_id = static_cast(symbol.text[j]) + 3; - llama_vocab::id token_id = vocab_.token_to_id.at(std::string(1, symbol.text[j])); - output.push_back(token_id); - } - } else { - output.push_back((*token).second); - } + resegment(symbol, output); } } private: + void resegment(llama_sp_symbol &symbol, std::vector &output) { + auto text = std::string(symbol.text, symbol.n); + auto token = vocab_.token_to_id.find(text); + + // Do we need to support is_unused? + if (token != vocab_.token_to_id.end()) { + output.push_back((*token).second); + return; + } + + const auto p = rev_merge.find(text); + + if (p == rev_merge.end()) { + // output any symbols that did not form tokens as bytes. + for (int j = 0; j < (int)symbol.n; ++j) { + llama_vocab::id token_id = llama_byte_to_char(vocab_, symbol.text[j]); + output.push_back(token_id); + } + return; + } + + resegment(symbols_[p->second.first], output); + resegment(symbols_[p->second.second], output); + } + void try_add_bigram(int left, int right) { if (left == -1 || right == -1) { return; @@ -2070,18 +2186,22 @@ struct llama_tokenizer { bigram.score = tok_score.score; bigram.size = text.size(); work_queue_.push(bigram); + + // Do we need to support is_unused? + rev_merge[text] = std::make_pair(left, right); } const llama_vocab & vocab_; std::vector symbols_; llama_sp_bigram::queue work_queue_; + std::map > rev_merge; }; -static std::vector llama_tokenize(const llama_vocab & vocab, const std::string & text, bool bos) { +static std::vector llama_tokenize(const llama_vocab & vocab, const std::string & raw_text, bool bos, bool escape) { llama_tokenizer tokenizer(vocab); std::vector output; - if (text.empty()) { + if (raw_text.empty()) { return output; } @@ -2089,6 +2209,13 @@ static std::vector llama_tokenize(const llama_vocab & vocab, co output.push_back(llama_token_bos()); } + std::string text; + if (escape) { + text = llama_escape_whitespace(raw_text); + } else { + text = raw_text; + } + tokenizer.tokenize(text, output); return output; } @@ -2670,15 +2797,15 @@ void llama_sample_grammar(struct llama_context * ctx, llama_token_data_array * c for (size_t i = 0; i < candidates->size; ++i) { const llama_token id = candidates->data[i].id; - const char * str = llama_token_to_str(ctx, id); + std::string str = llama_token_to_str(ctx, id); if (id == eos) { if (!allow_eos) { candidates->data[i].logit = -INFINITY; } - } else if (*str == 0) { + } else if (str.empty()) { candidates->data[i].logit = -INFINITY; } else { - candidates_decoded.push_back(decode_utf8(str)); + candidates_decoded.push_back(decode_utf8(str.c_str())); candidates_grammar.push_back({ i, candidates_decoded.back().data() }); } } @@ -2879,9 +3006,9 @@ void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar LLAMA_ASSERT(false); } - const char * str = llama_token_to_str(ctx, token); + std::string str = llama_token_to_str(ctx, token); // Note terminating 0 in decoded string - auto code_points = decode_utf8(str); + auto code_points = decode_utf8(str.c_str()); for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); } @@ -4132,7 +4259,8 @@ int llama_tokenize_with_model( llama_token * tokens, int n_max_tokens, bool add_bos) { - auto res = llama_tokenize(model->vocab, text, add_bos); + auto escape = llama_vocab_type(model->vocab) == "spm"; + auto res = llama_tokenize(model->vocab, text, add_bos, escape); if (n_max_tokens < (int) res.size()) { LLAMA_LOG_ERROR("%s: too many tokens\n", __func__); @@ -4155,6 +4283,62 @@ int llama_tokenize( return llama_tokenize_with_model(&ctx->model, text, tokens, n_max_tokens, add_bos); } +std::vector llama_tokenize( + struct llama_context * ctx, + const std::string & text, + bool add_bos) { + int length = text.length() + add_bos; + std::vector result(length); + length = llama_tokenize(ctx, text.c_str(), result.data(), result.size(), add_bos); + if (length < 0) { + result.resize(-length); + int check = llama_tokenize(ctx, text.c_str(), result.data(), result.size(), add_bos); + assert(check == -length); + GGML_UNUSED(check); + } else { + result.resize(length); + } + return result; +} + +int llama_tokenize_bpe( + struct llama_context * ctx, + const char * text, + llama_token * tokens, + int n_max_tokens, + bool add_bos) { + auto res = llama_tokenize(ctx->model.vocab, text, add_bos, false); + + if (n_max_tokens < (int) res.size()) { + LLAMA_LOG_ERROR("%s: too many tokens\n", __func__); + return -((int) res.size()); + } + + for (size_t i = 0; i < res.size(); i++) { + tokens[i] = res[i]; + } + + return res.size(); +} + +std::vector llama_tokenize_bpe( + struct llama_context * ctx, + const std::string & text, + bool add_bos) { + int length = text.length() + add_bos; + std::vector result(length); + length = llama_tokenize_bpe(ctx, text.c_str(), result.data(), result.size(), add_bos); + if (length < 0) { + result.resize(-length); + int check = llama_tokenize_bpe(ctx, text.c_str(), result.data(), result.size(), add_bos); + assert(check == -length); + GGML_UNUSED(check); + } else { + result.resize(length); + } + return result; +} + int llama_n_vocab_from_model(const struct llama_model * model) { return model->vocab.id_to_token.size(); } @@ -4208,16 +4392,88 @@ float * llama_get_embeddings(struct llama_context * ctx) { return ctx->embedding.data(); } -const char * llama_token_to_str_with_model(const struct llama_model * model, llama_token token) { - if (token >= llama_n_vocab_from_model(model)) { - return nullptr; +int llama_token_to_str_with_model(const struct llama_model * model, llama_token token, char * str, int length) { + if (0 <= token && token < llama_n_vocab_from_model(model)) { + if (llama_is_normal_token(model->vocab, token)) { + std::string result = model->vocab.id_to_token[token].tok; + if(llama_vocab_type(model->vocab) == "spm") { + result = llama_unescape_whitespace(result); + } + if(result.length() > length) { + return - result.length(); + } + strcpy(str, result.c_str()); + return result.length(); + } else if (llama_is_unknown_token(model->vocab, token)) { + if(3 > length) { + return -3; + } + strcpy(str, "\xe2\x96\x85"); + return 3; + } else if (llama_is_control_token(model->vocab, token)) { + ; + } else if (llama_is_byte_token(model->vocab, token)) { + if(1 > length) { + return -1; + } + str[0] = llama_byte_to_char(model->vocab, token); + str[1] = 0x00; + return 1; + } + } + return 0; +} + +int llama_token_to_str(const struct llama_context * ctx, llama_token token, char * str, int length) { + return llama_token_to_str_with_model(&ctx->model, token, str, length); +} + +std::string llama_token_to_str( + const struct llama_context * ctx, + llama_token token) { + std::string result; + int length = 8; + result.resize(length); + length = llama_token_to_str(ctx, token, (char *)result.data(), result.length()); + if (length < 0) { + result.resize(-length); + int check = llama_token_to_str(ctx, token, (char *)result.data(), result.length()); + assert(check == -length); + GGML_UNUSED(check); + } else { + result.resize(length); } + return result; +} - return model->vocab.id_to_token[token].tok.c_str(); +int llama_token_to_str_bpe(const struct llama_context * ctx, llama_token token, char * str, int length) { + if (0 <= token && token < llama_n_vocab_from_model(&ctx->model)) { + std::string result = ctx->model.vocab.id_to_token[token].tok; + if (result.length() > length) { + return - result.length(); + } + strcpy(str, result.c_str()); + return result.length(); + } + return 0; } -const char * llama_token_to_str(const struct llama_context * ctx, llama_token token) { - return llama_token_to_str_with_model(&ctx->model, token); +std::string llama_token_to_str_bpe( + const struct llama_context * ctx, + llama_token token) { + std::string result; + int length = 8; + result.resize(length); + length = llama_token_to_str_bpe(ctx, token, (char*)result.data(), result.length()); + if (length < 0) { + result.resize(-length); + int check = llama_token_to_str_bpe(ctx, token, (char*)result.data(), result.length()); + assert(check == -length); + GGML_UNUSED(check); + } else { + result.resize(length); + } + return result; } llama_token llama_token_bos() { diff --git a/llama.h b/llama.h index 92b474891493e..6ed1c4b916c68 100644 --- a/llama.h +++ b/llama.h @@ -336,6 +336,13 @@ extern "C" { int n_max_tokens, bool add_bos); + LLAMA_API int llama_tokenize_bpe( + struct llama_context * ctx, + const char * text, + llama_token * tokens, + int n_max_tokens, + bool add_bos); + LLAMA_API int llama_tokenize_with_model( const struct llama_model * model, const char * text, @@ -377,14 +384,23 @@ extern "C" { LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); // Token Id -> String. Uses the vocabulary in the provided context - LLAMA_API const char * llama_token_to_str( + LLAMA_API int llama_token_to_str( const struct llama_context * ctx, - llama_token token); + llama_token token, + char * str, + int length); - LLAMA_API const char * llama_token_to_str_with_model( - const struct llama_model * model, - llama_token token); + LLAMA_API int llama_token_to_str_bpe( + const struct llama_context * ctx, + llama_token token, + char * str, + int length); + LLAMA_API int llama_token_to_str_with_model( + const struct llama_model * model, + llama_token token, + char * str, + int length); // Special tokens LLAMA_API llama_token llama_token_bos(); // beginning-of-sentence LLAMA_API llama_token llama_token_eos(); // end-of-sentence @@ -472,15 +488,43 @@ extern "C" { } #endif -// Internal API to be implemented by llama.cpp and used by tests/benchmarks only -#ifdef LLAMA_API_INTERNAL +// C++ API, will be moving to common.h soon (TM) +#ifdef LLAMA_API_CPP #include #include + +// +// Vocab utils +// + +std::vector llama_tokenize( + struct llama_context * ctx, + const std::string & text, + bool add_bos); + +std::vector llama_tokenize_bpe( + struct llama_context * ctx, + const std::string & text, + bool add_bos); + +std::string llama_token_to_str( + const struct llama_context * ctx, + llama_token token); + +std::string llama_token_to_str_bpe( + const struct llama_context * ctx, + llama_token token); + +// Internal API to be implemented by llama.cpp and used by tests/benchmarks only +#ifdef LLAMA_API_INTERNAL + struct ggml_tensor; const std::vector>& llama_internal_get_tensor_map(struct llama_context * ctx); -#endif +#endif // LLAMA_API_CPP + +#endif // LLAMA_API_INTERNAL #endif // LLAMA_H diff --git a/models/ggml-vocab-aquila.bin b/models/ggml-vocab-aquila.bin new file mode 100644 index 0000000000000000000000000000000000000000..e06b39b5a31c1ced47e4891fc892bc956e97bcb5 GIT binary patch literal 1963875 zcmc${d7Pa^mH2%G1X0m;$NkD4WVwW}gFNgY+ZDnh>%HmjBn_SPolZhPWaa_fN1b*V zmvMVY(n}yDHwh#_2zl(>cA3$cc3)?_qmDW{YLDBD>%8By)DwJv^T+S~>kS{0Q%_Z$ zI(6#QsZ*y;)m?eZk|j?Ok*T-n-~MM0uYks>t%UM*5L;3yCpaI{Fn zfMZ0O1{^EWHlR*h1{^2yHUs91j2Upe$cGGA02v0HKwk|wQKVzQNg|IJa55AcaEi!h z3^-L})_~JQzHY$j5O2U4B0n+!Zp;~QrpSLAkkO@S?blf%`xtPx$kPlsN936XoJ*Gt zI8WsH2AnT)paB<%yv%?L;f4Vhi5z0UYekMQ;9`-Z47dbF8E~n{0s}4+IoW{A88rhM z@X&zQiJWV|6(ScJaHYs41}ucV23#ew(15E&t~KBqksA!S7S0>+dXWJGt`k{m!1W@- z2He1)7;vM=N&{{Zx!r)9Mc!yYlkqa3C324ei$vaJK)=Xa3>aW24Hy)8hXIR4-f6%R zk#`%gluRR18x=hfC0k@fC0-y{?35qA|EwiMC9WJw2=)1{z7Dp0iz=8 z3|JvDVZchn#(-5K_Zx7V$OZ#ei)=LDb|lJxJ4Ch^@CK2G47gKdn*nb`*bKN!%sO@}s%eA}TdBLCvhR%U@i4~cx&p@&7j@6a}pA2_s~ zN#f8WB0qNMQIY>}=rNI>Ikba0N^y|8n)@^tr`&{Ph+S!6GV-XgNMLvLk*I`lS?r#kd@k$oL{hsb^o{WbH~p?8Yx z@6fwMp6<}QMV{f%-!Q!$8WTCdq4$V9%c1v*Jlmo7G5Z~QzsPeO`dg9bI`jdN=Q;F2 zl!HSb5_y3`9~OC`Lw_goB8NVL?r`X%A}@C6Viy}}`+ z?v)N9bv1{Ox>q@b)E(pyQg^UJNZlb0A$5m3gw!485K?!zLrC2b4k2~(9C}dXNQaQW zS389K9pw=6ceF#u-!Tp$f5$q6{M8*o{*H49`J3+$@^`#L$ln5okiQcgLjF#42>Cn7 zA>{95hmgNh976t1bqM)8%^~FPbc4FN$QceHf3I-}`8(4gCnDA>{9ThmgMu976ssbO`yo$RXtKwGJVF7dwRfUE&b(cd0|j-(?OVf0sLi z{52dx{$A%0@^^(p$lsL?A%6=ULjJCD2>H9(A>{8GhmgN(9YX$I?-25LokPgq^$sC_ zH#mg+-RKbVcauZN-^~soe@%ywzm`MD-y(;QzkY|1zX6Akzd?tPzr_wAe@h%f{+2p~ z{0%vT{N3UZ@^`C4$ltI-$lo%DkiX>)A%7zdA%AU$kiWlh2>Baz2>Dy#5c0RuA>?nB zL&)E44k3T59YX$YcL@2r!y)AF4GtlHcRGapz0sf<JUfhmgN~_dddk zQFIH*9p1fe8Y=GIy|;Z|Df!)2WcQ|N8og)tzI;}=yHWCcqEJiZUcN{zc(6fi@Yy2dcWiWPfau4?~fvjC4avcSvQSn{B3+*B>CUk=WanW@&Wriw0w!= zA4muh+YdTC+1)02z!P1D;6qV$1neJjszyaV9A6pg{KNKjWEIxa9`3G{{O=s&INX(w z1T6!SCvX`?Rw}VU@_^SMF!C`cQKjKyUSx6_{(syW8tRw)%tQ-;fgqN7;!j(^0 ziDBfaFzD!`C3mQ`Lgc=Xrq*)ExX-Jtn`W}Dad>KaljLi>4@?B5WUc3n&)h3{z!P1j z$T}|thqTT*Yitx^#&`EUNk&ypeR_=to@tcEL~3oLr%-bW|lIO)?3xV<;>Fc2|LB_H(L7ndjFoDptK%H6hF~@K=Md2Hq)rZ4XOTi$pfC~ zB3KU^u0e1GG>jb^63feS@u9M|~txa1B` zPjp$TKV{#$g=)!9dG{FCPdol8E0cfPzPIS?XQJKdb+9A?*x%BL3&2YZE7Vr*e%Wx`ByJ7+1(;}!0U7+{jLx39g=@HO6WBBo?}PwQI`4l z4C4^$SibKW*bK_=?>kqK>wojg6Ld8Hw!+pVv<_H3Od1{{Fk68DdHP_tX?fer#V?fc(dau={m5e-a4k)=xZRYWhA^P5+T- zcvRKOe*~&&tbw0ePFlTEYPI$HXWlp*{h8O8RN4Ev!QDbFwVyj)mSh+#bJ0q7rz*@y zvA$@(|1-78vQfd9$-jts)vq?dFA{>yEVlAXXCbTMFa0~CsM`8hK>#a#{;TNmK_b6S zmDj7<|FyR_rPkkXcGv!wwK}cY>0ilzW5`w_Lh@fhuByZTa>Qy5Bboo5awa4XcxoE9 z{MM))!Ql99G_9uD?;IZQqOX4EcrYteKz?sIC`u&!_nxzEddKtzJQ$urC7=fKNR3Z- zP+0Ou&l`u~Na!D<_m|@ANMt=I`JaNPZ8cQ>C)L=XGM!S>>$ES@-NzWy8kS7DdjYpt zMgm$SrF3m0fbGf3B~P&ugUcj)iq+da&ia!(+$~lQN}0avvay%fRl}nrS){P_b*k)T zm{u*5axcd;IgOC+ZC~4M$@cE zi48gm`#aJJCUUmF=cozGu%ZdzAq>LgH(_6dj zvY%~uy2aq_l07>rcPCZ)J!kg>ohzSh)mvEA(iat2XS2RN73j6+V1_>-Wx$h2fIKJY z?#If_p5t_{?~Y6U9K+fzE|jwRycTDkV`YM~r^q6O0jJB-b5pl>O6f2wIV$LRUWa7> z`N<;11X6?MdB%m|0V(4<>OuFj&yVIuC3}AA$QIRM&+k4BX;&Lc|413R6WujxhrS^7 zV7p{5Fx0vSc|pRk5u(c~wT1ncywLMh@|fK(3|XL=VVZJgJdrX zyy#O_JI@##MQUFZWT^A|Krf`qk_HYmqK4212U;0$GEoooX4MPv;#B+zG-&o>%NSh2 z%yzI_oGayvW6)aahImQPbG~FRG2|otlE1{Uu+4yHWu(UGBa6sOqv1t&NcK{zJTyEk zxqse0HN9O*|2(QYpO;yQ5n9S#=19kcYvilF}*$s8P7$*TW~)YdM9u}mq19|P%?UVBRU`O4J7dfmoVDV@b- zl$%gxQ}b@DS}s{FP1 zW>iuhDWn-Yzl_urYZ`{b;i)$dKy_B7Hmhzp++^a&Bc(jtsOc8VY37K)-Mydovm-2L zXo>2?BcjjD@F=7!?t}Q*BjYm3<|WE@sGZ~)Bco{1dEVg(L_&^?@@GhPWNPgJm6b@b zU^7e0Na0UmIg(dXRO-|Mld>rP1Kscx}YCsKZt zai9%(QEH-VzPO`}z|lcieYDd(2{ZGf9Tgib2r46mpuk!=#;UaFYIclQ8An9(W2}Y4 z(Lcwe%Ij5^9&5!1ndI5ADPxW9IwGZ35v%jDj%xihYQ7#EWP)aOtE1aD-3ogO!Q7y1 zixkegnR__1@LE%<<^$FVn;+MGCOR7{i;V+IkZ!oj_Pa;cOKV@Y7|9q%oT>(U$W1R|6>+%1^rBjMdkeu@*OW*~PpO7fk9+xLL9TRF0oai}|)02o-7Afs5!j|&bP@s<^#l}^I z?Ife7&FGxu7}z^89{EYu{`~rJQaU&~$og=ym%(a6)n+GKnISBtJTk_+TXir`cEYA$ zr<@X9?_VmJ!y~I$adU@ZD7x|#BjT8YI37wJG@W#+H-KH8pXw+l5$!Sx>Hdz>?SKI$Iue%$?n#y9C^gyLrE}~| zufs-BRY0Vef6R#d%;1E|%$eR>-8gQgx3=%eyoGhlq%2E#c4A26p0j(B1+>hJ@>RNq zpXCV0F=4W^yu!pxu6~KLoc}1xGBR*p(5{>ved=!1r6W?hvtz@0wxe6G^DJ^yl`#v> z3FOEjR`)r^-w}37=Xi|?<-<8nms+gn2JzEdRoEh>wjW!cNU6PsnR~7ym{fs2FJQG0 z&x;19H|cs4DHV+eq>L0hd#nRF--ua)0G%K8nPB<(MohQ3Rm$@n)%xyQMo}*C)cEv9 z)ixJ+l__0nB8OS5y{XEXlj=fer_O{6t-T?H;KGFJ5w-Oqg`ZsaK^H|HjbgJ$iYcJV zCQ@j7wV5t*63ln_+JteP^5C_h{<=5=Ohgdv*BT*h-Gw;V8Xe%kv->RD%5}pFum4JR zal*D;J$)Bj%~n4gsWKi_v*;2dV5ndBz?XOxwkjA&d8Bj&WvX9d9l!Gi&QiUUt}z=W zyENFYrxF!R$1V->)k}-ynk)4(uZizhW&E-jq{}6bjLGQ@*f-^6Uez1`rv6!tE)RpL zpOc;Ja>t`u5(Dvar(iv{O&K{;(>v8S-0*U^&{az`tOxUsk-QNtV^<`Q-R|pB=||{G z7AclnU5g@xS6X*xuX7=I1E<6lB`oV%q}0x0u&yw=y2UbPVdPBELA2MEUQ4%`XqGF3 zgE;#0$iZu_!f|ELc?(C+*5sR|kw5%&ztxE|UCe$AXT~jeDe6gVg7EO|-xmGoIIY)6B*En&^59 zC(1Q3a;w?n%C&~~xVmHxt)lR?R&PLs`dTMpg3in99XW%iswz_0{;J|5g&=iVR$iYv zft+0DwbajWk7U<*r`3xZ8JxwSnafDwM^deLy&-5VLbqqvN3%4K;wP2EDRh(7 zQumVXrEW6z^|NaE_ucsS@M4ycLTvQd=4Qj)Ez~%;Ik>NGk^E+(det!2q=Q5FTBI2r z=x%|TSu-kMgg-LDG+0I{tV~sKEw94~fXaQ#@R}3D!EW(dDIIL-im)g((a{wlQo4V^ z$cfY>lXsv415p7P zT_<^@)PIOh^HjIMjnO7%B3PwXK{n_Z)WMkHV2_4ULa0bSyV)%m_O)S-GIv47N|oh=Po&A49b zHPtl2w1|`*Bw&&cnZ&eim2AjMVJxdjF=SM5P(C1K?{#nnJt@>_x* zY!ke|ZuQQ}tyXwsm2UZNwZg}sUvKrM)TU)fBc&a~tQ<~cY*Bl6*w7A9Js&nyXYfbL zVW*lwS!Ts`i11Wpo?(gG-3o`)uCWUoJuu09+on4Y|qh7^lwsej1)GLietOGpZipo4q+>lk*VetBk>nr zaXog+qmo5RO+WU>e-XpE2=!d1f=q3yje0e9rMlXVTF<)0Rgy=Bdc@XAS*3Ppr4`)5=hGy@_ljUB5NNw(7K*-jUwr$-O@N@pw~73E4JdC1Qc zS0y5!&_P{gt+OV*L-NSsC`@%*q$WA?kD1@$rN-ePlOa+Q-Hlr34Uxxj1giH9(YY42QVz5Aeku2)*rmyx z(JqTHe2Wyugl;lY3dM|*_fDrtt${Zh*c!!s_r|1SvA5Wgn?87B!nqxtTt-UG5Vp;C zB`SBSJVi=XKh|u@>*}&t-sMbNucr22dM%YPyfl9qIhbYnU)o>|;?b+hJc10%U)gv3 zc-dbC16!PiX|8Wr)-LOQ$~(%B)3a1QmR*>?(QJt)l$3LTUPsx_n<#YKzp~>}AH)G>sfK>?qM@Rc@2I z%-$U1xamPirjf+^SAlY-gumKcui8WdT7+A zb*^=r6b_Bz#P`y>Cm^%;79(Xu&kNpaU`t2jt=;FUMhg>vkN%SEt= z&Q&jis~a;UatJ-5cJJ-ZI@FuIBf5h*GbOcm#AvKQo+C$9*Q3((9873TiGU{%kK!FM zxv0M*8Q4|LjTqRyDz#0QvA=f0hl#|g{dIJyyFopYDTO0sN}7=xX9K`e8Y#S<%&5X~ z;DJObdX9Qd2k($x!eJX9d1v%y;Zmu+Gx~{T%Xx03R9SVTnNrZPj@V9P&L9L9DFZVe zeO;v_Knj}gGJ-}{Em=yeXzKiS{5gf^pn$PS z-(>_0v4!a+WHa|Lk>%adpP|K4dv`R0alnS}-BzWAsnLuKw6ZS1RcbT)-y)@0L21X) zVI*&sUdmy8W2OC@U=}B}Tc!3lhP+#xAX&m6F8#C4+i?y-D53CSU>yhjG!rRZ#db*ZJ<*f~P@qjMj?=3S;ma=G6E-W-HpC|P zJ=QEn@PzaN&jj`jYteht`l3_!z1G8HkCp7b-n6b|+ok#5pn4gHab5-`QutRawMi-c z>v6|PFKSLsZ-z{HpVcHnZH3g{m$%$lm*m*u&X}zJE_=$*8^G+QdQ5*5duq528-CO4D=LfU?@9oarZ+?M0+G$v|0_ zUaqa@M?HrLkmLOGZ=(my&2g!DiVe9PzN9=t0Pd4!r0_wi*_%?#Vpi@*aRSS5l%DDq z1Zl9qMjk6Xi=BKR_&z)&HHV1=!;tzwH9BB#Mrsm;%fc5aJ(PocA8_vBPb?!tJ+^D4 z_kkGqWg1icLB}`VRdrmeQaja3{-Dvogh%K;xJSO%?~a51585!?zEX;)H-&qYb@oZA zZPH6A_KvuBKNNgKk8PLQhrE|>@4j8KfD0manhvvu;(+;(w`|1g%tB=pA4;nZgI-oB zJ1FH|q&Usc<~|(FO>d)G?ZeIz-5WaC;&i$hWz>3siIGw>b4rR2$1p4(k@CYKYa`~8 z=mj#)`Q`5t6E%#p<|&Q5)J;v~O_+DM`FDm7oy9p_sx@;r3uzH4-PPgyh!jyJ3`8%L z>`tkJ_aojKK6aE$EmGqsSzP&%Vk^%;G(Y03Q+sk#9M$C?F%$OxZG1@a5gW|KY|cL7 z!#>UqruPx!*dXo@`KW;{B9m(l-ZqFi@lhv+xE^L!Gg5?LdOmwVY8fd6ecDLxrWP z`w;6!87X?l81x(yiVR1_$Gt^We7mIfamS0X*nOP!XBVm}ixPxbqx_pGg?eW+Qwklu zS&B&M=?xRT%2O9p@8gNTESI0~HjzEtSG7;r7!Y`teIk0Orh0-`5YX`nr^@!*Oa=|$ zD?VWZGdjpr@S9u<4h!jPT zawVeSF!Pd$=H*w7qD1bqs>9rGsNI)Zco-V8`W z5z~XgGD=`V(~aI*3%X>XEb5WFGagFJ;k!N4vEYo zrN<~}lyye$=pa#GW#p)#yh(bVgJK`&-iJ474lm?f&#h|X(MD@njqE5vaAbEKLCbm` z+4K@v73xVIvZU!WJUF*-dT+fk?D#sjbW*E& zDXvzfHZt0orwG#+K@OUcr)N4i^ioOI)p02zg^y|GUMcUlc837+NRMQaZ@9V|S|q3nU6d}NU{BZrtN_?JgY=frj?BBgT6D*HeT{xa!Rl{T|v z$Oh}~4L3?{gB9r(hfB60dXBzlsoM}O-pa8`3UZ5EK)dQ;A!*0sI$l}GdUwHWQZYDbEAmj8lk}Q_Mnlqtc4);B8T8F zUEVfYdo5iC9Mo$V*+%QBvTb8Dv_eByH>SRBREJQN+RQ8{Q)=d3ES26yYmfs2OoUBo zOf>YRwkbHYnDwm6S;J^GH>IXDa5u6vl4!jYRZ1^sl#!amFSS#86%xH=B^_@}6%s0| z)vh8<_e2*4=7{_j}t`}NQ)Dyqr*s1%;jEe?~?K?GiU2{|(u&*x@GZdX4+ zt{1*)o4t88WblSYiiJ_P0G=WgemU3t^3B$C8;4Ed;OO9DDK{H&L!+Fg^`f>~9-Jp_ zamcS{oj^gZVUHi8MRpyIV0+&R97!_$yTf1 zEl!l$*5DL_oo)5z$MH|#Q;8I3C>Wx-m%Byn!6NF6tCHFpZ4RiGN(k5na(??z%2JQO zL($~xC40!5VP~%#ixl@xG#dCJ=gsmZE2M}V*|mBIID3(R$Y*T32;@1w>HwgSyZV{j1xSQ#mfv2^-Iim8JQTR!T;iB-{y3?xb~8$Oo! z`MAnQm3kE3Ws%a=nmtpbCUF+xMtsbOIPOF#s_$DMqKuTDwQ#QRm^C{xJiwhvZ%94q zEK)nHKmEh21|*9dWem$lD#voj0m_b;BqNI#OR>Y-JYGt#j+J%4^iq-KgIw-=-0<#R zk20-29%5|^rdn5=V#YVSl7|53LfG^=-Qp}3tjIT2M!tp$7LNy)QHwa7a)PNe}4NTf7IhrMQ`beoS8 z%`@0R%dsr=!u922@Pjxd*#b+!JhB*n@&uUsGsK`v@-J8n0nxcgRXM z9qq9-A|hZqQG}O`HC;W4z4%7VXm<(&W%DQAq`LnG|QdkKXGO52W-lj2XsATPq}^C?HiWJa@RDd(J} z(u^F`i8At2MxaitA~FaTKv|TZ3L5otRg8SELb?Hp<t8r;LL{)qOhHyNIdO zj4YOFW?BAeN317{kFr0|OH)OZm|!P%pOlf(89pw(Pg}264Rfw3pD}99OZ}O|uMM~( zvd<)ra;OPA^txWt3%3R^xf~qCPh5N^C|j)I=79t+0aidSQo0;qvVJx-u!;Rx?XxaA z9JFMgO&#FMggFWD?Z!7Xd^YV4H~(ji@ZD3mAc{byE(mxVIcvLXrT1B*hXYtFS^4`I zHS`gOXMgVvnLi6L)Bu?5?}Pj{*Xx>IoOOLAYgQh`RhbdVqYj!{o5`f8GXuaCkBI^;0$8?9iPB z94kH_jSlEG;qx(kY_>4qy=J#KiTnL73X@nVHHS5(O0s`Uyt%KtMwY z3GY7!6IPCDP+b|dbZ+AW^HjIEWtGPI{gY!<57wmA{wcRp+SCzF(Ewj;KY*(CLkTKa-> ze2S$o|3Z>7y~^VAs7R#=MP%jo(rh@?5Cy%)LV(_AT(c0mLkRCNT7c%Ly^pgVTKYa>yL6OE#_9 zmz^e@)!=JJif~j6=e)|>#4=x`iY$8A1?88WCUtRmIS%4999F(!8G1{+_LUeQT=h=V zqmq3k*t}4KESq03R$@`&ee)cajxFqYi&T#0Q0QMVcA!m}#=R)0cOTikeAQ5lSR~9> zqjN2D^<-a-&T&X{w=}ENMh(@B6b?ZSB#M;Bc_9(3Uk&E#eV<;!IWV*sCyIP6`lJss zn6h9IiErivwUv^U>ucUNN4|P;7b(utbaDP#;O`&74oHcmY?#U@r-vJaUwkdNa}Di& z-NXe!u6^CUGM-$5__|Z0r;VHlvJFKAd_A@NfV$qi^pKv$7ExMnu5%P+PQdc(K}!o; zwii|Ptcq3R8<91N6;bL9{8 z{)UmhVkPTP&tVYY+xez4gx4U2Lv;3F+YNZQdw}PR(#Y81Mv=oqgLosJ9+i96YC65WSyQIU6xsifuBu1{+syT{XPw}vp%?hwuvD{WBhSiM?huOeU%ho*eV)+P z@pWd^H#J3lYt;N#6Okb`(f&0CV|u$TOaE%5j;O!0j6x{wURCG`q*cBfebnt=S{}Y@ z{X)~}Uh%tzW#t|0E}P$V;%?_{8*8*%U=4mZNr0ZrmQ_lld3u3eIsRB+m+$$w52!Y( zeJ@qkNYm_lPbNm$!rZ7zbo6ldd(QrGlztI88upFnvnsa>o!Ik6R(TBd_q`F~r8x7e zecv0I(7*>KhTROVzUlTg&Jw?GO<@aQumuJUsM#vT_oMTtNcnx^$Xz!`&!G|ah5wdD zLGM%6A~lKD=UT$QdE<85X}3D#Ff67qptby zjzPC-MAk)09bo98NU3tb9hOq89uG?|aM=qDFjXJpksmvLdkZNvLG8!EomRhO4)+go zm%aI8Yga9L4e<7q_FX4T%43)Zr2MflYUmB9gdZE)nyDO?Grl)3huw_3=qFCEIvXFM4|?Ui{E4@QOQrV{m!Ks)9_3AGTpkvbryyOe zpe*v#v4Au9KdjDT!uR95Mt5S6{ztIkD(u4qufVW~Quv-ARel=sy@Jq;+E2YvuB)kY z;iraRw+4DPBS%koIFb0NQM`DFwZ5v4L#h0!_fAJ>XXvzEg>wf)$)n7Y@cVaLI8)Bn~%EiVz}bzS0K>Y^nmmt#T-}l{c~>$ZNX-*_VeiH zavng+er_z|+^hNX?t%Ynm~*$#|9ET5ktj>?o2C?UOgv_!boau3^yfyg3WLn0PTCRU zoHa&(U^W*cvU^OLsSK18ju9!1@4zPKcl6YyU|>=l)_5q zB;-GXZ^(>Zp>O`D(a}$gq?h9Ym_GGWRJC++mQ`xAwpNwY^MCn;@lYTAz|8(d>f~1D zZT5>jBR_@m){yjOlwdq{7DP&wGM=wrxSQ3~F_NMYt$qYhR(SPwi=H>BUIW}Fzl<@w z4IfJFm)<_n^*g<=6g?Y6wOMwazNyl%@DQS(9RYI+AC8oPj?0i3;=Nyn zt+aA!DA8YeYn}*O);F><*!TzaKEyh z@FFJ@u<+}|%xy?+?bn8Wl}5tBFghssnP2_huV5f3d_XUT6}^fZLnL3J2qzx{J^^d@>UzNz*02qfC!U3@cxl?75y z0E*u_**Zw&Z;d@C5}WkfBqR24!S7-ySF7Dq`<>Mo)G(N=%F$=Gnx0}FJTODF_wS;M zt5)dhTtudtQX8fGT_Tx#NJxd=@98;W4wW(9es83-ZkO8cos`n|WTm#Vg5cj_$x`Gd2yAGZF%P_A+(*VO*57qXj-%n=#5Pj_*wkU%r&4!%4d+!h4qN>;_TGXcjrn2h))a#`fXYox}@NL)~`odWL$C&4X?4^r8ChdENZRg8BT6c$*pUgZ{Q=pj1(-e+xcHNO2b*I*4W-rQ( z+%B0z_(U6liQAH_ea#9J;fN#!vAu<8*GT#V5!@#pMjXPura(d{zxRPgg29+Pk`D%ue<>zrQ8kf(Up z)fKr_`kvxl_x-oaSnVk@ez6yi!N4l1rzEi?I?bPA4R95f3DkIs^=5?o`qF%gac4!p zwEZib#2w_t2kphKv1nWEyh$l4N;=qfok*#AV(S=m*_C_kGWvTqN^swj3eES zKkA}UYftsg`y4jALF(}hgvJoP_K{Uc9z*jrJf#oaARf)924!5yk#-cMuc+ZXQ0dzs z+f27i$ksGz%W zR@;43*Eb=!efvgVHNM%*Wr7#I1V<4gSPu0lf$ZSSh-U)!joxV_bz@%>8_q(cxvw$O z?%J~YwNtTW7+HC=tGfb(df%C+xzcn8Tj5BSpw&A28ZVBxfyZAB^DvIZ*`UHaRjh(~ zDXEuwf@5Zb%vE&qZVg^ReTq=_o2h9l?|AZB`hUXGw_i*<@Tyg^iB-0?pYaJBUvK5q z_lq76aG^De;uapL?g^A*W53XF6mBMT`j#UjtPjkz9Uby#q2v2mABl|_i?a6a2Dy6V zRU};a2|D{34>SOw3>1tb*7R)E2v3dZ1kw#al+bI}Xz0{~0ll*(PxEnCAwuMz7CqM6 ztxOZWXkB|+bYWmf>Rwu3q1Lmplo}Y(C8(;aEn954Gy}VyJ$9u1G$$HIZytqp-c3gT z#-cD6M$p4WLdUZ57~E8sey1v-*VN0@yovLyBX8?%_9({98mzjxNa19J;j+K?S1q;& zrEhTYx^7TId!h@pN5_L*sxF^+0OQlf$rDkXn*4f^0rQzfsbdP*s8ZZ#ctK! zc#AysnQWD@s^l)+twl;*IQL7(>1e4p+QFg4gEH%7Fd*~={oejDHx><=CuuHWU@t@j z>(diW6VmtekkJ(_BHU|F_wEpJGR@Z4p02BV>DEd%)mhY>!uV`>O-BA9wmi**hp_&Y z(tf(}s>OB9JW#S*(TEFsq+=|qsyZP^|LMlfi!PInqsN2Gt@Eg-8t*aHx!5JMj`ldc ze39Dg=MY-vqOjTK2!P!xdxrJWPC5IYVf5U1#KBT~h7T$SEhv%tGpy|!kK!ri)OFr2 zL?rd)u||}|*P(V|q&Q?_ZD~7t{7i#9y&QE-wY%aNtJ-;(qxK?7=OyKn?P(>TC=nE|Ok z(>gl;B;J5F=!kjJc&4+0Eh6Zeo?^eRaWHMK%R{^aycHd|+0F3c5vC8zP1NPJP);if zFTHz;CAbqg<1E2E9Z}WgLMP_PIBvw*M0mg65=}&~IE>5OGedAWCbf%L2c*&3#MY!Q zQfi72s(FBu6{qw@Rz1F{GE>E(II{z+!vrEU4lps$rM?*jb+9&?QrRYLN00ciA?2~_ zm>C+$D`U~LM%*B>2N-8tqnMwa17hjdXdV11soIK_oF2|rH0mxz!x|Y;r(zVv)h|=NMG=(Ju5L~Gb2%dmeZnsw(MEKz6ddEL~T75AgaSG z?B=sfZ1fO&g|wfgZe4aB|Ns8SQ{lnHl}q()i7}^#>MvZ(_HXOS3XW8pY5~ZMCp6BYbpZEYR``8z~RV4Qh#>Lu4T-C z%uDG}K;zlQZem8+U_3iXpk5@?bDizd_M#*GLp)|464cU-{8*r8j;MR1LXVVm94C9v z$9eTUCM{}oHc`aNhtXYCLBr2QmP!fH;C)tU@hGPleSO{y4a-Icq!dQ>R;l+{Z`GpA z`hw$ai}q+lISoWuE6qOR&mGI8?cepKHfBv8r8J~;i;Q`zttBwOh#W#KbYJN8xdBl2 zC7F)fPxxPF6Bal@a9?VH+uK;0juTc8g7%ykf))#P-*bXqG%of;?KwVf%a=?2IYul| z;MsG0Droph<2k8eUL%Xi)O=2wI2&L_+e;tGA6|v`(jEX|+e8`j<_8f;Tr^Qtt=Bcu ziQ=Y+SZ_3PymX3Ip)6*jFm6p0V$a**y_!`rm%2oh8gI@!IcnIT+@5RQIHfMN=UO*V z`5mc0*Lc9qWaKT23Or?w;ob0J{pvJmJ~#E&4~W$r*M4qrY|)_l@A4>u?`B%>6|1Q- z76{d^z{W6A`ijPS>3E8fx0m0ewl841{T-KCr+7YbJx(RI3_-ke{+U-H?L5TX5RqWpJV)%|VI= zG2!QZ;mkq*7dzW-A#*RZ5j&pi>Mt_r4xDo@GJ5SUZS6(YDUIH#zbGVmrT*&6i<~*C zGtynD#*1PMm+8SrGZjCFyW;IXEACGLR6bI+N%9|IqN{{&H(FO-+|r(4F;TKc-2a)9q1!9!R)A4 z`TXPtr04_iS@q4m{pyx##5bLmNGApcQK<(erN_&yZEv$%oXYQK#CP3tj~(c;q|x%@ zQg|u#=yp;o%)UoW9+=F3_6?TN@@ybb+2;KuM=*$%noG@a1i+7T*tqh!@5SDrx~LK5 zNR4;bGI#1;>c(UE`3~bjx7dA;G+vzgfkE5!BHE+$`Fwq0FeR2^&y9IymUzxpqX9j} zd009x4tDD^Uvx@;$D(wML~YJi#B5<+^&F)}t>80HS<}RV@XH}KfGvC@eJ`<&oy4O0 z5+AlPfq=sU=GzMgp_X`_QSl6zHZT*~(=1Yw!i*c_7VonmUHF8HX zxLhsp$WaFskv3jq5vt{-!7j8PmQLvzyg)D;4YA_tq3?`76VgkR=)IJ=ssJa*%oTgK z{be>%-6E5|m(6&@A&4!;y!JBhrdmgn+D%rHdVI6BDYjEteDl)~zgE(CnX!QDBI-zu z0?q=R2=`0d3vhRc0(lhha|>Oi9+9z^CH8Yo>}AG&Jq%(e-SLz$o_HMZ!4fVK&Zc5F zT+bOr?`6j1yY7@ZZ;NAM_5Z&-34xxb^hHXoF`_xoW8h^kH+1t) zkw(HW{TTDR87WaQ@~ zvg~2{cov3`J?qhUg?GTO*Ye04OJDO9#tXelU~XY)zruTjHv!SlBefpi7G7b;&3=$Q zN#y7Y)a#@ZDLvug=;jqMNqE^oW}{ULB*Rsw!DYSb8*ffy`sqRLTtdP2XB|J+<07Kr z0DZ5FVMUMO{*IJhKH;3h%Uxry6TC8-J&P@B!;4vH4L87EX`EZKMA{A=e>}hY6SIxb z5<;%V0xyp=qnwLK;eb=OU@EHjoVd*R%8&$BE(@(^<(iFe%G{(lda2qlo_-t>DT()j zpc5|S>EY>u6Qr-^_|&L*9NExGRI4Rf-lRK~nvJ)f)n!%2PBI!%)gl{NMC~SDeDej7 zU>+94c0z#1DG+Ex$a)!j5;_d`Dz%o?wo?m+F2OHb*217%F=%Hvvo#-~aR$5>)l{L~ zFLPCD2h&Dg<(<s|@uDz4Mg4YUa6ciHbz- zA}R!ds9D17YSmt4lr|^eKn2$k8wb?Dv$yLx}wztqY@%H^-Y6yg0f%0E)8nUy;J7|lJeS(Evj#aR*mems$H_XXpfWQ%GMmc5)Tx+^y+fUu7#VYamhCTR$YIf|0grm>J1lxN zfB{%LEGT37-v~Z^lcs)H5{V5Q@?}*@BY_)-8M`7T**Wb`hkr{{suHST%1Cj!5=A+iNM43M>K$g?U4`sNv>E%Z!no(a~_8{E9B&kMDuq4)Z1Y34;&u&Zo% zQD)^zefzl?h{LgGHxt_PVm2U3>bcG5C1KYqVq;Ye9xbuyfPFtNIHh3%9d8Wu{M??e z>wfRbb|4{wvsH&UCIpK<$JTQ!t1M`0F43tsP1s)_nFNrx!k7;F5N+R)P8ShgOuO2V zF}X%Yb%g6j8h1|B=WCC&fgf#gKe=&aum=C1+QZTe#P(h^+Fy^0+toK*yxK)JPWh2G z{v$*SRL$BIoItKByW?eAZ{SA*om^j<)E;J$t+4VuFIY83vlrh~Z~MKLhh;8(!(*dY zO!R$1d3DTK?|7tCt_c}ddKT+tE?Ag9PA&n@Zr+aK>qeQYB^cl4F zt9`Dl;Djr$@;S8)7tNSs81?|SBH?2b_PBIjy~mi_sb-wHIMv- z(u;5Y^q@cMHW#&YgGLPgQK6rXJ6ZaUvN1X6RH+@6=7$BJX;8A%y(IIUP0~>z94nS- ztUx0nX}6^5NRB^I+6f7R!7^(Pdgn(4C^Besr1% z8(6)wqm4y~*kjT-+GggE2Sj->!fEspe>)J_lkiBs{%B?HRq^zp&W?_g0WNX+S!a=I zz~6N?FYb{}G&ZGoCxQ%}o||O$=+IC7(u)i(Ng}Xwi41&MTueR2dV2@~Z^szj!+Q8w zJ0=DcXNX2T>CcJR+^{|?gn^a3rF^;v? zSgbU7sqa`LRA1p+B(-Bx@3c<+pMsSd1#~MqHx)V2rk zF0k8Xv6Fi6fv|Ztj(sV)gc}^#O!W{*j>@o>>Z!Ykr^*dRWmdO75;D_>vg~W&doxmc zjJieIks_v6b#UOZUtR={T6<&h&4bc(W6r+5?i$Ff9kb{+;w#nP2b2Zlv$@rFP~RE&|ejutEkc?P>%E7sqxN+AW{=JQ#hhJE(EfljZ6JFZ*DDulO30s z!0!bjWSk;OBYiV(lN)}`3i(b}wf1qL+=rIgN}fjzl^L$sTQtL1e6t8)H5JwR2rO7& zI`XL^VYQ1=pQI~ksfavITu zr{j1l$59v&^v4^$L!dpkWH{Yw7h4KT2ku$kwt&(mOs` z3Zo;75YIU;#7kPTAm%HqwQcLCKEa6SUrM;`3D&>XB7a(7EEOBjm>e&r0Z$z- zrava5N9JY1jg%7V;j|6jDm{meUm$Y{k?MFkF?EP1iXSaRlymeon9vmGvoVF4snR`)wDdo`$`)xl)ICv`_VFUCmO5m z_tj5wF4)6`khC3hvoF43X+<;`9@mphaG97K5kx*-Zw_4i4TJ2YG)35hp@$k#Mh`Z( zS>Jf_>m*;w2lb)ewkIs;j~`}^X~Z=Fj#*QdI)aOnOva8sSvpPvZp>r(`Q|k5I%->Z zN+YA-Nw3PgACqz}rRW;rN+)}F?bj8|#(rG-PENS=LET!U^!ENeQco!?Zw^>fZRSpg zj%%29<7A`n%7xN&h|BKnlZ{#3LgRZ9%$w_DUSjB0DI7ZDXz8RDh|uH5%ucqEQQzq7 z$)TBXD6iCzy`E!2sjrv0sHK;qxcqR6_dt)fINCWS`e-ND{YQS%J8=QOm2J?er^;+13J(ar)KiQdssZP`$g7API5kaNKit)acM9k$d@vmUX}AjTj&Elt8vn+c7mZVxBWwEbp)S6A7`E@?*zJt5Kfb$xj# zcLJEC_|~GdMnT}NIL$h`dbM<_db;IvS9h7dQP%E;%-~&fTpZUEjTX2jI4)ye2I2m) z-s#>?^JGh5QbBFpX zdx>Bc#0i=66y8ev1=ce{JuD^8;|vqk!!@uleO`?h<_zzzKFY(@)iaFlQ%;aZ^&O*E znrArMxH4g(*ZLC{?K6UHS1ujo_uIXInfJs<>w_I*slW(Pt%bMQTB+g0-b(!he&{__ z<1B~`y*}JM8!hV!KQ28dd+AVH=Ay9f%O7S1(Z?O-HI6`coDb@F(r({ttfwt}N43`& z16Z|KOkNWsF|3yfvXsJ=Pb8(MkS~@IX$DdcBlO*7+cBA^Jde_PNJM~PDs6jiC9Wey zRYIR9?o@5r#aqXOe{Wc3qpnJuhK}pgj=c(>O1v)59hrMg@OtIQh{&0)tML5vojG%W z#WoKu-*a|cJ97_D^jfvvIjo=Q!>n)U6Fw0olwX^f|N6T#jWdl^cdXE7)*@dwe8dDX zfgh3fnWnk8uxqAZUKLqmp$po`n4=&(i(t|-O_Gj3NjmVr^BJ% z>=8Dnvytrz(T)hD=cHjD6Dblzt5+9)(pf$_`ZUK5={w6?b{(a`CPZY#H`B}rif)hU z@y+iU`7iXQ!PZcA~1S z+Sy5f)k5&=2=%k03rqFAk|==us_@y{?TxczDx^)0oe6kDC=2wPw4<6*t5&Q12wRlU zZQn*2ixL|A%P}j1!$$pVV`+P7OFEIQ+IXE=$=v=pJA{nqZ`t+Crmop5SUpeSrPtMd zF51F&P(SH8X-e*71JoBO;vU#a)y}atnbT5tP#?_Csxmv#6AjPM$2ywl1PA?1BF6X} zn+4tC8ZLP_ylNCyk5w2n^sR`L-Y~-}d`=QOJMQgTWZF5wb^Y~&*(i?mv3U2QxL&E; z7*{IiyaZCFF)(tjjrTG}_+0O*ZVLDnhI6fR-Qqf_dj?a2TR4(MF=ArSc?~b7WH--s zcahErRf!mtq1W2y#w^tneZ)|Iza~#em+AXVV}Vo^^SBh}+Gx#-2LL;%Is>XVc-f#y zd2aA-;i$}daUKz{C&+r9HMA0%<^r$A3vjRHJnx|fR-J1 z4(k1o+WFP=W^JuU30*Dq!kzwZRTkgki3EoAA!(c+V!;o#Is(RX6Oz{U=7@B|+Y&ka zmKVg21sc1z&wMn$+WFRF0?JV0ovQd+1h4!vTGb`Pna}yws}oOe>a)0rzFvO)aTkV0{_kNv7O z)Cr4gFHo~s&0P>eGK}L|E=&We$3i-5xEFe1ObN`W+J!FZ8iPV~u;&cwEX^W^z%7h{ zMx^u@jrTz=v?drs+$n9(>J|(2w^j30%{(1rUQmBC0Jpzm=UG)Y+CbUQW| zq^QvG+Qw_GSznj>JfT<7mP+llA^4m{s|eO#8y#Z2i4c0NciHZrs$ije8eSchud4dB z#vgO6wxh5;7G`8@lHBuKi`2w&)SSQQJ;)-V#*54E0-)-Mr%05w1_*si2|VpG8>MX( zH!)=^6r^HBb51cADLuI*VESU~#;9AbeHVu?we5+J+QmLyIUlNDZ1me(_L;+6QsN-s zVq^31$4T?zG~e-~qjdD7qJ43U;s`&6k$Y2SwlZ%$MfZ2H87(5E>%>FSaikb)Wi+Li zD`qbacdh-tiaH#7(WI_#PZ%bmDJo3cIpUB@k|^?f@aRpAR_ePXM)E8k=E4zHyTnHl zOOc&Zy-IP~Az9Q{rKqR<8(@CJOX%Hx?$tLh2_0}Q9@I;MearRj1j#Q6&3&zmUE-7L zGASJ5MdXg-K_lxdD-)g-x5{kLpzb813lc?4I<`C&MLL~5U`Z~u9;kAFD}9$n{|MmF z{d4V7qmZ&3XkF_1#)4$COO0GT+0$DQ4aY*cow~M0O2f){6vuINiz_jl90%e_pxoG{ zp~f}p($9&Ds^%I!j*OI!EYW+Z?rM#_z0?QCOmvuQ&z*X43^omYo{JLhHM7HHk;{Cz zbdnLPd70x-g{*He>v>J>GVi1r&GpM{5LW5$%{q7+Dyk7k^fZD%f=DTnUDB=^P-k9# zS&Z*xBQh3^*)Og!6?K6sqO|&kx!CO~HnF;(l~La9Vh;asZ_Zv8Qp_9gajqVvv_l%E zrO(~YRW&tMjux! zJ)aeg1t`(57;j%M8sNulnM8Au(&I$j6^%5CW(r#5NMFOnb`5J&tr1=4$~o~7jp*8K z?8&o~MFaw?Yap_{IegRD`eq{}U_{!HZ9AAy(|IE#Sbylx75`YnxN^=yDH^eZ&>MOB z3UJ4p)0@WJ2riw*ZgCDiiy%aw&mca=b9g|H)2dYJa*Y4ic~4mriIMRr_rCaMKLKIG zw_a+mvmWck12EKI=UVm@$^1LCKZ^Elyv}(v$r99z9KBC7CGC_lZ&4mCaHwSO%<3;- zjlJ&AtOGrGU*LkBnWBmbGCWkHS&v$sWPkohy=?f7PhHdQ_%t{57j&Hrk(@xd(OA zbNnZtEOY5Q{&X&mU6}+_13vq%OoPmf$4t_@&b9Q7S*{1r8V6R7Z=~=pvk^c)WPwR_ z26td9yxYOij1-l`+))kpg!Dy<*C7$% z+Ct}*hKp&ujQ%E7eWA%jdzh!7vxR&1@b)7rji|2S9vnU|jKxk}CfH7G&(*QTr;{&? zf#{oej)Y8yjY|;0~`>u-KV=1na+Evyl*A%3Fl?m_({1(7fKD+FpgGS&`3E=Ht_Atbk z$2Z+9Zbe1t55%?O8&(46r+NVuGtv>YhSe=I=1nbSD^nyI@DXg2PE}LiMfYR0(rfCq zWZZOD*#Hn_%A<=FIqHiYXBu#J5a{%J4Htl~_AcSu=({?GLx1LqT}McEbx5%O+Favm*JbPR^E9KHilMSVSM|2%bDT9O`PIf!wahVi#ymwi`$go0 zbP>M;)k*D}bh+cIbaGn+%Faej-69h|l4`cOlrUG-)Hi3@-Yhndt1I(}cspTTq zMo%ri$35YF*Us<)EpF%kZ~sZ{+K`Djg|1)g^MN&*P?RiMQd`&V&UDvEBfe>9<;KuI z&8pTzaJC&OgoRm`M@rwOva9AxeA7Uu^e$dAzNvA#LEU6J^t^~83I8diJ%3Bz zbw=M(J(IZ3`Jud(hJ5?HyYsLtTt6#8rUC0b(6AtT0 z>FY)4ysErDVy8m{AFi_zP#={P4if~)>mAqG9Gth?-}xxL(GyRU*+9Z2lI}xh2l0^n zbs@O?ev;~$IY-1RH5?LO@7+@6d$)0o1Geic!G^5b^)cP?=IC)uRN#6SXGf~RcSx2h zbax>&jq8nRoFqvzWpJ84Dedb+gm_+3^2pLyed2nrcS*Bfyp9%hjd)mFU@z10e0HOR z=Oltw4r+|g?Df$*4&U_2nqF0%SJ7D%^y1`P^-WO{bHjv{8$vaNKS|xkeK$C}mlL8{ zyCJAuh61angl;gi8)7LZc+e;ZsRmIL#Xm{d$kQ=Rw z-Qo&DVsA8AT}C|YjcLf&Fn#McI&1tku{~}nnU~;qDOyVi8I2OeaOl3y-p^>pHx2C4 zV;u`r@>&Gv=npXFHwH`3VTm1!;^SSvs;fV&UZjQ@bUpdk^Zicxrsp|KBaQbegIw<4 zBRsP=8vFJ4;CfMbk_`d}&iKxG5ih=}m2ba6fMqK;c@J%miOr}7vwiW6Qg#PYACRua zH(iSO&33!^S-&X_+BzD~5+?fr%_-C$PQshg3Tcl^G`)ge5N6G4-(*Z*gjE++?e;6Z z{M;o zxSoEqb7>N#ODKrr9J)ou;=8VsoVZ469Pil{=|oBo$*^8;4ry7588aJAXxys?u;>r= z^c?Ax=gFM+MBn-37dHJ(&5=cRnBUh7Iymg(oV6J^)oQ9omfmvUCx@HXZKg#dzO%M6 zn9Zhh&i((cj@t1}ZFh}!(ch-bn=$BYV^FMPe^$-pM~e97w{diapvjxT`6CHNtI+am zAgBledSx~d1Cbk~ml|2dxe`y;rbaYc2|Dyjv$VX!>YvyqeJvZ~Rk&1Zk--$ASiNPO z;2jO{WUZM4S^p4l<*quD^wtgMNc_E-p=dZ3yt>k?uxti>J5v0F;tWsMwv2^$V9Hjd zwnKH%N`{%mQ0vKUr)Aw9>gQ@?=`d~?)-P{K6+|z6K8y2o$Gm{w1zePL6Ook=th?#H z>e~}`@T)K9)Z&}Pm9bu{53Ih(S)+P^U>HwfciN%NB4ZSHl%!dqV-tsRK-#3eCy*W!}gRisL@bd%$G$WK0IBBGq>mD z=*`i&iX43bPn|2ANA@Q!=u;IchJ+vX2Z?sSuHU<=Iuv(l6xUc6`=KLjuPnZK2!sFR zd!wq=(al=Fk)**DdfwZ1{QNFEhJNmO>`$3VTI9M|R^i9tCKny3p6sCq&Heg~r?+9rOm{91?dpDz^>N5>x(hD~2_@?T}ZGLR{ z)be?o}zEt zP=`%N%~)Ef{Y{~_C}3 zAi|pQ@#3sugELNKuzB5Q!!rrHycNJ6&qEko?@eUFIr7K*i%98`q*7Hdha^fP+__7j+z|AkW z^COf2s%m@vWAOTD3gYpv($0H$Z!WjXg5b~h23T_O!!bIaBvRn)-s@8{OF;L@`~Y(Y z-__h(b_L0yp!xC-YUjJS&_)pdrF<8LWpQ`v?9BtLj^*E8pHt#|tW;njK!JFyXH9Xx zCByh;zH>`JR+O{Hw)gs4?DqIM*2sPQDx&iYmIq8VY`Bq=PZ*Jxtk!T4&?6HYwh=?o z>8gO1KJJE(dx4y4lcg1WT&>y2x!Wr~4R?X88+y1v)}`v*MdW&6ZP$rL$a}#--)p3( z@s8v)M(O7wHd*2@!uFtE8AzJ)4wfY8R7Z zovh#i3TIA2(mji!fw++58 z2PxCQ;r9h(ng!gL@=58SPuf^1c)uc>mWTc1oSZDp?V7b_{Ek!@?{g%5*MqhcZ=Fnb zvQ_w5pzpmeF~b!p=x^UwHb{i4*R^gvYjCI{>f6i`+~tu4NK1%vs6bHPXZENsj#4=d zK3WzPbr>k>#@P9;q!QC!sO399-f2=f1jiorIhs{Cs`{fz(iB=lF~lb0W3vfbHme4) zY}DprRP9IOVEY?edbG@_;?Q>jwpX*xy(sd~TWsagL;(DTAoc56Lzh!@xlyA59o^0k zM1GZ@^uB1ASkJF4*y<%&tYfbQBTJg^iNXtj_vcwIV5pn{G5Y)PiAf59BP87 z&0%%e+WUOu@^wnjrKp9qx^?y?+-PzU67{kX1suXJxHR&*nyp8jk=OjKZB~VjA?V&N zu;F*xFxy|Y#W~pq?e{5sH4d`-9p+iolC}N05F}A{W$jpb!5Qd=KK1>vZ!2>?L``L5VS1i*JnMfX3I}lTWD+m}rb|5vtm{^Af zxSrq9y<(6wd7!A&Noo>aM$IgsQ{b$%vkO-kK~t1!Z>j3`Eb;R~q}DlbxBLVJ0pQ9& z1)J3hpc|Z`D+iKhJce7vdVG?CljCmlK$4cXa+iqTC|bA{&y{^}MDYybNuqga@#f0> z$ec6;HnlmQA4FSYbH4l@1Zyt)y-Zn2UvbJvdk^kMl0;96MNcOpha~9#3*Xhsm4F*2EYpgRt)5%`6Yo3ta7k zK{O%sOYuP_Ch#$xswy7PekO8Rtg60*uzV#;#0eQ!-t~jI^X54GHxK5bu(RbkxLrDs zp2}?A7aSqOYi)4e#~`baeKR&ZU*;uDZ=`gpMF|Ly^6-t7@-pYK`h50-EZCD6m9I0? zXsw&`3DRLeXlK)*!WJiu4SoUS-P{$Tu)+ee*-M|y8zRnplKtqcO9C`Ka^s}%4k9&PYY!gL5x_HY)E=w6Z4vqAlkPueR_ z-+tOA55>{&h&2x-wIHA(OhrCPGKTyzpTrcgjiX&$pZ8#0AB`Q9qHiUB!?C}~)}spC zw08Pt)!cKiJKF(^qNm$JwM^)gf^rTnIDE5O?%{>pxzmV0{Tfxv8lpj6c=C&gD#!9& zSkAbp+Ug65(lBipjAaQK-6&PC99f&u&nq3I^zHJU@#@Yg!Ol{ACsh`hyx{XR`UdM| zyHcByIW6Abhm0%VA1CbyiQ5-)adx2<*)D_N2qQia!?MQ<>Dz5QEAm+{gWB!9QL)4eLydM%0_J^n@6GseCydf4c~s6 zdKf9#X23kVm$PR(eox7j!CGaUm!T7bEpHw3IvZ{k7_lT`7354>s|xp!Y>u@OiT1$y z)LXf0Iq4^}6~f6hvlZPiV53^)-X823?Vy(lF{}` z1p=0jw%)q^zy7;wb=%BV_CY|mqXVfah+b>j92}E^a;%NFiBS#y5vik{kDS)v#>1)iBc9_xp(WA4P^(Yfl`g!I49r?<;wzEh_%w)EpS8>)I(uZa0b9}wW_5(64rp5@ArlDq#N4> z3JxWVnLiN2*Qt0h5pnPX1@~nVG@KSTVE zz!Y`}+Izy9A8=SWZr)(+=n3xYpyJVfAhx>aTJ01d(Do4@Au7!tgbJ^eE>kD16qD_?3FCvYKg5M(+=`<dg}TkO1z<-1Tao_bln81-oI&(p?ZG8%MG z!{!ls)_z1z;ZWv+#rOCL%em53us)Y(Z0?~|9`S8ERfIWvhh<{ZCF6t_M zFRM28Z;sv%zQyYQmVnFfoLQZWFK@cfn(^zQ8Ecm^@1d?#t^ejzjiWOhXZ)VtD{Hds zSCTWiRVYruN`V>1m@_M%IMsD^EKeSlbGs_(@s-;#29Ioz;~FUMFaSN~dk=kUJD2_b zJvP)XH0?ALYPjuVtleuPS^3xK zv6+{m;v9x)gmzqR7skOub9UN}q};~#JYn6^5N?Xuw~pkK)^8WV0zs)}32|#H*%6`* z8IJ+sWm~K2TtZ5=OieIB|wDyN%#|F`H!Hu6*+fG!Y$Qr7ab?* z__j_w6(r(5n|vzg0;4;&WFx9-XG^pn#h(0EYh{x%qC0NwY*KCqy4j2WTa0(2vmBto za4$U->n^@Z${)p{iv7G?0VRqV*S${kfsR>mAG+Bif+sMbSHA652)6!Hj&5$6mc@jn z7i_Ie!#*Mwtw$+P)HbS_oaK~a**44Ey?LK)-8Gu9=OlYUd}kh-wiB~3{-D{@v7kPQ zj7Kunfv4}t%-n2t@aez-oq(+H(>~L0c%u#Hdmx4B8Hx9M5d=P+%lMv0Y&1Ivy9N(_ zXk%GVW;Um7ymaTu%E5_xR-swvv}K?%5sfRD%+o$oN*UC6I#Hxa9KdEgfd4Q!t*VDB z9L?>h{CDrQ#cW!v_gQF_drB8*|I^>tax^}NZDgkm+QX%GqlqU8@?5~>k}qkA^`p)d z;;@wh&_e*aRyA-{wn)(c3Yd-3-{TluR?1zcv(jG-e8g8~YSsonl6!#gAb6q=eWYw#*)xN?R2=?DA<;eb2_yN0i3TiC zEuM%ypa?Ld<(pq|$MQ+$D$m<^J`smix5)s8yTa>j;v;bq6WV7oUQa!W=(3R&&^^n4 z{Uf=84?t&IrDb>sRH`3ILV}YOsWojep50A)7Q61HD81UjvTE9vv#wG`z{$jTS35)> zLLK2Dow@#+p?lr=NS@ueC@6m=0``xV<4>d|lnnzPjYWb;p%ohZ zXsk7gE*tu2uJ8jMe|`9)KHE?J>j!Kk+mU7(!!}greXC_Vnx6AE8trfb<6D!ZbpsDe zp}vhrX)QnaUH@p_pj|ZNvpwRy*;2lf13({~mG$Ow)zvkQfliEhL8P_xx&U8cy#gLWMe9c?Ibg5E zdn}!KSo^4NmXTN6dbTQZ1WS}pVr0(SWE% zN!)Z@PMaS~OnyXRcIR#2W8U+JK4gO*%Xxg?0xWO5ZsB78v9dKUVoWmfvA_$tzJ|v- z!ME~VM8Lw<5ry99#~eF$VPU9t^~1# zbPVRZI0Ym7{7_X6wV1}Ci%!1tY#e?qx^5#McWjWxg)|klk0;8!1Kb|H(*$O2nrGbA zt9MdON34MH(kyp1IkPDeNj(~3iV@jO!~^yOuxyiATI~tp#0xb(o;W-o?w`%li(J!^ z18RLdkNwxa&e{dc!wCNttL_Ov>`caUwv_L5wd^_TeBG8mo(nyN+0$O@lz}%J=pVcB z1UyG|4ga{~bm4ze}lb4%k+A4*}ROcRNi;KSA>`Bac%lHn-k8imYo!S>y z*v?{*5F9MJi;lo#r;(^QAFv(WWBtWk{_wzFDNXLy0@75TW1sK|!aJ1) z$Kw;(E_9u&M^6&xO;qjPf_k{X)+RsUsQAx+YmHC%(qO)U*<}t$Nu{n^D_Td|!SuJY z5iy>U!pH`GG5UD&Rj;z80_XGr8A;i4vCf9*G|+6!+Indjjo_4Rl&3us7;Kh?FqcJ34qCrtTiKBE72LAzfbo@Q zxsmLAA`xo8OJD67U#GW4 z?LCVeDjMRd$kdKj1y0?TE<=akSoAu(hYd6SjIR~G1@(CQ%F~dwi2!jwa=pm{)_&b| zqKpR=Z9a47dgi9ct*R>-*a;P$vi389#k#T`wZ&(0B6j0nfm$wlnZk8>xvEQw2*z;+ zIZP)&o8D#JQjc6LkeFW8=oN0J{j4fA%2BtdMZr;khRoSo76=Y5E;8#m*b{8djcA=D zGoB`9v($YTAZ``7++w-=@=>XWqArfJ?Ad2>U#9tleX{J!@VvnF+6F%9>pji+Kln*s z?>$qP2@QQRrv@@j%Ail?>c5sC8Y9t)9H3>?>Q~8HDex#K!ssV`)d+t!_Q{;mxfz+> zjej!X!9<$jSUo<^?OSk7^h7-F&+u=P2?D|~aK9075=zXP1vbpKk60^ut8tv>@PvHw zsPC~%ivbu0hAhoeRMcsM%wst!zG2ilQHYcoU{SYB5`@mx@30HKd?#6oY#w^dNk1nJ zxx`9#;#08}&EALIL3Ahg1_<{?yx$EUReC_gaSc>dLW8YnO#z6vqu~5wwi64&NzKLQ zQ#p5pRnoaX@TsyuM2ceZ#ZLCn4|*aKKXWMRaYoeOlAm+pK?^)ubcPMrySkxyVsU0h?S30oCmkuC`rnB zxT69uW{>BrSK4)5Wmv!V^PL+|I!H%sr2+;4%37HKO#HD~twwMi&p+42<_I>*WK6eY zf4W)Pknj6eSxO0{vCP~qZF>HPo$^G&DVF`yzCCsI82EHq4|z36tjxMs?;_9b4Wtx9 zpY|12P+@Egvp}Tv3enZbryZ3fLs!-pt9`mKHVfglF`AVm8kdi&_mWJrIb>|nxFQ?TkSl2&{|moCIRNH9bXWQ z+-Hl`&=9w9(^ssXr2@nQAlPyYLl45H^ovGoe7du^k1LLjhd+pJo85mLGpzs4Ng{3`H2iZ8!NwSpA9|h)BUgSwc%%TsZd?IwcNNK)l8QySrq9t_V&sMW8$xSN;F*RfDEPRr#cXC=qu+Xxv-cfSe zXL8M;x{upHc_M-oaX~&4#O!r8^qIgBaYgcEL$zbWQ5FE7=MFs*W#9F%)jpF`O-gxo z#%LCUkrqQ$J@sG&$3Np}HMM}TR@9%JwTXDC3`>)rNpwMAZjI94%a=HUn+Z|z*i!th zR+b>CHt@ASQz-f!Q>I`K7pu`+Mc$_6Be74_-c{+T8DZa+vpxBe5XQY~T{gSj&p0~H z%KEw&zuD>Y)-R(cMk&N;r93w02<&ESpUEx5p}$-C64w(*u19V^jBARM?&G$R0EzBl zs^#Lu%?e0mN`b_K?OUY>?AfNUsS7As_bKwu?qmgB7r6Ah=ejNVq|d?g48+gX&fnYM zQX%MW|Lv9vLrdjYKL;rvUh)w`4_{{^`31K^X1bP7AS_#LG@oR&imtzO@gHnF8X%Yj zabv2cUx1WOEIFd>!k%Txk!Ln@m^2b9C;JRo{!PsaQp~^o-3@EyyRZrfd2M?skw?PU z#nKejjb@Kz%ycgwNoYRlrpa_0^QGs}0b1-XbF^V^i z+Is1F53e`k^@H!W&1|1k6ezH^s@|lv+pb0`{1cTpYdcGMH;qshCu zJDLHHYQbr?$6DD1?jM{B?E(M+VPyugSl}mCn=O?GWYe}>Km+}bTPL2Bn38qu7C7Xz zrF)my>8c+rtgK%qa=xjcV=Gxf%qteAZvM4ay(7pB)Wb6x$6!cV+q8{n5LRGH=F6Y; zHPn}kCb1Q3xG;}Ld+CkiQ&*%iC2z*S=5w)x@?{}2DN-G@@}G+%aEU!P_&MJ)4|4(j zTvBjsQZXosCqLVJHuAaL16+=*mY=xIb9#(^E|H6B2!xpNvGSdWqo-{Ab51`0>##vq zFO4cT#zerh+WEV`v&n=x%mISdHu9Zpvxzd!lq0-~I(`#rWv7ZXgBxay>dAkx#XI25 zqak4NmP$)-Uy8*d3;X4$jk>l0bKWU)CRr1xCv9xEY9D%Jy?9y(KdOFy1Je<&uH=&p zcTU;b=X|H>3b`IYz3Fwfkqgk}B-<>Vi6dp3AnLcRXabH&^1bacghO-rW+%H)vd8Dl zK3@*iFpTVR3=QNvlH+-5Zv~tf{Jbx9?Tt3{`Pjo!6hV+b?;sxJ^c;x~gt}KWIRm-#NnsU3Dey=N*3zxejSEd)KGy)`(A2s4SDj@UgSx@>j#o4*Soo}PSgszJP{k+pS_xP9S zZY#d7b`EQAZU>aR@B!Qj;IUZ8iunr(F#)<*@_{ep`3__12^$B$;1h@qA1A@k7YbRH zf=ph@kmV@wxEo%MWC6wR4?Y0j!7l_GBeDBcwza#sxEaT2mX>H0FJs-0Z0rj@v9r=4 zkAESF_Wmhu;;>j%dDPoHC7?;r-}+vn3t3 z#hRj>J&V#TmYzL5BRq683-njfV=w2E%q6Z_C!chwI%(YkqnvbEUR}+5Spo$#XKp|G zA@>IbQZC~}sSAo!Ave=*g%%{c^E9)GwV&A$GPHc=`| zyyiT?WPW>tHNIE~jnqT#8EwrkW`+qe{7mngYlacV7af~-Pd8l#*3NpcOk8u~3|h>0 zQl28#05Foq#2{u7j@xp9L*B>a1!7a@Amoz@J&q?B4IM^ z(+Y{a7Qb@S~I+9P@MrmehX2v#44<5#Ks}$sBhe>C4A)a6U54yPR#611^Ys|jn zi?Q1I;0JBsOG#=$OEOA*DQ2*2HCaLoRqxnH+D_25;jASU9JW*WbP~k`@iF5`B7y^t z+vt}H4iOZFi@;d6MtB|^7l|>29pHG?+7hT2LE5P_fmY1Z8TtQ9{Ccl)#Xun6A zX?!Uthks-*iGGBSX)_y!l7oY-s+;HOv|SoONJ+-<#j2kWrGo{wl@>#se=$60a zd*D5&hqJkpj?~>RA%b>M*YvlZc$^6Mfcu$4an89!WZn>a7=Wi3mGE2%Oj zT(tFAuXjq@v=P72!7SKjK)^*`=ikdxR{fLh%z8P0E^cF96a+V}fcdTJS8R2LmS`h>HgiTi6oPj+GjZ`Cwb7*4AW6dlP zn^M*_$E;O`Dmvee;Ki(mTwPtrhS9Oj*8M+Z+Y3Dn$mnkp<9QD&dav>zkz#YA!^A4;%i9FO6awA)z0M(sNjuS*?_oSs^g$x;gq4$Cll&^j}G28~LX(8~;in zI5NZ3C7}qD>e-0461hkH?le&vjyw!LW0R%1>DF#*WPva>*Hz&;YX)Tdr}xZSE1;<1 zW%QNY^%08}aZ3!x`6W_>S5H@HVz~_UprT21qCwJje z8c$Okg-p>}0_wMc+=c7aJ7um0c&PNoR~*k4q?XRlAqovW`c_V#bDgYJnqcV@dW zVk7Z!wez24$y)oWV+;H6jE#P^5CAtO0|Rl9`)Xkx@Sup7;dfg1tB$<;NoZ7h#s!C-`!VW2dy;8fi9!#*$JP=aNo(X3qWNAg zbD|)$%#uP2Zj^T}_1DdWFg0Z?K6r&w=-O7iTkYUa0}Dzp56p(5H{xdRWQ|af>n{$C zU&|F>;c%gVQ9)Ck?~*l!N{}e)Ye~8HcskUfuf=*})TQ z!bmNFbxCOm4_n;=zLvX7bGeUe&-m9I#}p@*q!{tQ6rux~sKCM~*hveq%?kKZut<=M znQS9r)2VXRnpr~r7h(}5?QMN6_6k_s{+cgX?M=2=07EN|+{<_Z(!K_BU?u*PCA%yvQo(hmxPOP}Ge?5=nv_xlZqo)NC zG8@k-iVClI@se=Wt9P>52nWxJ=w|iPtIybE{_fm+pEbUoh%!Ah!y%7YroiAfo}r~} zl|u3>K(J=*9G|53*K9GX;EU^;9uMJ>?>y)cBthBVm&;Hh@~G~4Uysu#4#|)rIa6)j zn4s5__bOWwaTU+shl%Ka-7yVYWm_p7;cXkf8Yn>%Jnettie(A6NCK3Tvj2?|tDOfI zh#9b1E!PG1=~e<2d~|r&y6)b#vm8QiOiOo4yYiHayOP;A@=QJ?Cy#+|#CnEDq^Lg* zt);|%w{*x7kwHr(*!JH@njrb_K^ysok9GDSel*{3teJkyM!(@$139SKShi8-Pl zj{PdmZ@ZpU~fy4Sq9c1iiZiy+hwj z3bA_XHf|c(1m`?P)c)5-zUfmjg--CW)xLT6y*W4o z;{_7DAY8gpj|$k69I}Z5)qWy{!|hLgGjMI!L2Hz@7-_NnZI-sM6Wc8bXDiARn917Z zn@ge)W-Z3I2A+{i)nHG;i7dy!VBer$oicJMoYwd#ur376_EnKEx>Po4I^t zv1}!KP`H6(wpPKe)Fk3U@#`q?DVMG&=SCLNc_y1popGDli~^EewXFi^gOUK`CyB2v z+fG*27EmTt1q`-t`I5S8;fxJ@E3iUVy;nGR&LZ#_{Fd)uCT{3kcdsX1yC(NCeBrqb~^lWhmBIKIyo*ZsXrdRP$uAoc7V{0}t3l!sJw) z$(H%qZ`~gGcW$Tu^?h}DT-*v1evR%{@*RGS zIF{;^!DAp=S?wqcwXK&{gtpunHp=M9+v1Xz&48HLom<&G+$~(jcB}>wai?sjG})Zx z{%@Vwk@yJNKqpTCM9H$rS02j2POcBM=}-eEzvEc(_y5U;v$Sv~%wKNVNWK&OIi5U0 zE#JxRQ=to=7afRR56%(Vd@O$No@OG)vn+A4xZn{02|ZrDYE!wOTvcKdojkc_6bx!I zU?SdEqm!o*X7ijz%_v5vIx1Torx$ZGGuF-zIxlb7VkrhoK$_A{;+O)C&!TNwj^^L; zpmoYnV3ftG%a5sBx;%`jG?%*QWR?gm{iyfGH`q#cplP{kYn|LQ4*=qN=?#wQ&@{f> z8(H=wfiTe7bZmpI$c1*KPTkmEM;+}#^L1gvpAs6}h4jk*885Z6ZF;D#!d}0t~%k-KBp!Z;fb7F{#mqH=pyxBSe<9s?kZ{tGJ7 z5(DO)1F?}TBg!S_;TZT*JL33q|J!V|boelGg|RHEhGZO|_D!J2ptBF`Y387<{Ug)kqvAjvXnAyatt30EbxP(_Hh$C*;7zZ!dVUS69zl>M0{9a3J@Hg0OUMZ ztIcWvJ+_3h{~WQcSRV|2_JX?GF&hVGTeg#s#8L?oH}QfD*sRaX zIho-Z{hoBrhK}W$zMfQ@@tYHuh0mH@G!TRuI+j?oN0BMA8M?jwG}Pxjn|JJ3nHtGP zJRm0Hl3IHFm@k5drP0{yrKz`Ks5KG2a4(d_^R3Ts5^`NwdRZI;s5g5I=N^+ebjYajO| zIehrAEyj%CID*&1QdR0axV@Y`h?i8XFvSz>l+o=+KQ}8|$ZiD((i7k~23hS@?;?kp z?!x`!dB6~;iY>QNig&#!mQtxaT!Ce!^EoYVCRf zV$Za^O(zNvgg^HfKa)}E^+@O{;CrA-vvdbpkOSx$?fMGfqwd|KC2W`W@pqe^vBeXP zVJIlksa1VRluhT$0ha8aI@uJuRF|B~<-8kBDH*y-_fGikn({Ce{b(1X4Z;!?&|KLG zRprE6OW>tuxoqnv%C2g{o6~}AL}N;}MSR+9Kw4E!0?!t=6$SP>6SN(@5%B@T$1;j} zR|vwyoh-|SeF>NTry=qKCo6O$PO=T2%%zqXO!AMR@_iriRfzn0(pT#BM4&oZSO)nJ z*Ge&+i^$sJ<7($0iGE{aCzJC-jLn_V#!m*SNX1OF50vBzlB&pV^^?8`|PSpOV*6{_xzK!vR1KwAkl}l^PRGJd$?NAbusI695V|`CmrqP6oD#0 zz(o!cpM?<7LM^)4j6$j_rhzC(FG@(AhQ^~`28yqu4zHCm1fu##wBAF@vKH-Qs>403 z%-a1p&mFXl3eI$~V54lFbXUwyj?VLovuR23moDX-R87V(tVI__xM_)I0 z`|f>nSUGon70Glr*v(xV<{~IJ`#IG^-CQ(sqKQKpjuw0r1Y?LNMYralt#y6b7&K8* zy5%%c9w@SnAwe0-cigbKfW!@ZJlj)vN&?koiNc`}D%K_n%*6D<*{t->f}OHPRUTPl zvJTCB=d37nM*L(e-$k6XkS?^Vmil1sV%L}cp?AK`ma?#W)j;SK)oZz%8$u3+5{GoE zLJBPwOiMRg6ul$UT0S8)z!0wAEgKA>DWP3s>fd;;t=$D~?ByG_9#yfi5gpmc#i7}Z zpm4;w7;fm%%KZmzt6PLb9zh?$#C8m58mawG1*-&jx;MRGb}H93Zag+{s;t-9%93XA zRP60jxNKdsp;I{p`=@1PfXCS>$4Qt63E#6UHa&18wNpNuvyXB|uNw9+_1q5#x|)SC z^<`gI0 z(#swfL-fDu_cFV^eJadHGE3CmLOEMJ<)}Qn9~bVW_>Fu6c}%ul)Yx~jPIU@&ho{|5 zpc{QTQ}&GYs-ABlS?LG3GMgvHP7c*Qg25yN3$~W;q?IW8t@Ug|=O^C6r+n{@-fNqu zV(Fh*{Y3m{Tc;A&X55@_I~xlt%?MPzQ|5P77Q1pEA{Y7TLuF z;OW2z5zt#UR6cug$lUO0$A@N9c3>l=F#>f>!}Zlp2S#wjj-C!wi%lL|XRH)&qTWTh zHhwyXpvVER|53#ixgJZOd7H>)N)*8}NR}t_of3Gw*+K@V~Zn+K~eLr@ye}`2BX>Ksp6@DMUCgx@Dda&ft9-B|*d! z){ieT(1d?l$vza4(Jva}0}6m#?_OX8N~g1)EgV6rypi8!<*tYeikG|@leK`xMD${- zfU|$<;G>Y@?E((0#Sz=7I=I1Mq-!-fy3hE^5gG(V@W7dzdMuk^RtC?+qU$nv!iLVo z+CfBIb}U1s;WIveZ@_aYzsiK@mh)VM=gfDhvt+0**yx#@$afHNDQX`=09&anF{cJWED`3#{GQ8rG>GB}XU7aUG3FH@ZFS_E5b5^WjIVA44T- z?gd-PYR+sh^~X5f*0M8QGs)#u0e6es1?W22XB$-srrb8m>O8^Cb+2tzgT-!lyOg~B zG}?`w@&tEIj@q+vHcJ68aMl;>HGg4)@pj+b6u#tV%L#BqXX$NXq}cFTUwmbW6^%ce zm?7=~C4BYhMz*8n5i<-fTw_sH7dMO%#DEMfoIw*BZCvQt@5A*_S>cPg!(#Sik(sbQegIcP6~E_ zHt@W|sLTL^&nKP7I$b)3q35eT&75mOFWd0*#S&vy-s7@99m!&1LzJTMI!AJ?H09jf zH5+}tV2CK8jTH#?(Cv6B%a(PT{=8%1>)&J(*|Nuv@dF!Wd#PHdokl5ufUekMH)BBZ z18qf1Z+w%rOaFVOp~beC^&+-<+S3;>(Aa6xm!D4x;rB~a1lGx^;_lRaK8W8EI;kb= zmF8A4AqQb)M&6IYN*T6Neb)UV^rUO~E`ok&>M*m&cMt&N`C&}8ksTdJWk&emc$z(Q zNDyvS?aCP{{N%Q)@-i~t$r57iSV2S}$IZ_9f-9ivz`1g;N8pRWbL9YzoU`X-a395S zO}624PAVQyXzY>ry<2e9&IMl;<{oZA!*$)rQ)k;DR%(d)f6j8+@x5~WCA0QqiK#TEI1xF+1bTBTWu4%s)H z{nkF0lxvS%xE8ZSupANup0cIrWS0W-EywRyK8uZNr_9z&7!Gw~W?ly@z4D1$9XEOF z$KV#+>UO0XDJEKLF#&%&V(Wi0YoSmMCT}C>mb8YpnE<+>1j8cpn{Abmhlkj9RU{mo z#F6YI++B`JJMSX~gSA)#@x*K>ul3;hSn;$kmTjyJlq&nHGb(7@1?jeb5?8J)C^pO2&FAkmxZ=ksET*i9joP9j1Mj&Gt= zIlu;=j4IqC(W915VE{-Jnoy#raUMzPh1U6U*&$0C(YFYnSe1$(cnUAHSj~r&^-D1_ zxiH8aa{0U?=mX={DH8=M9kXscaix*dX0O0FgG{7f3IYi#w`gGDQCrLRvK~Hd>($dO z)?_2bbx2Nfn+anV`+pHJF%4VibK}56<1qkHFFo#;bW2|c zr*rwUy2z>sHCZ66pVdVK$t2ZtGbEiPonBDJ*rtCtTOLCni=H7)u&u_*uAzvD--d%H> zJ1JF$@|}kQPx;R91)r@c82^zAF__m`Endz_j+1qigY7yvh_jlFMep3kB<&i%;8VB@ z;l9=5_iE>@`)nd%l)M~6*$N&Zhf9pD5gqO&ruc>2(P`rj6Bo6ShlyitUvRXZA{IZy z{6bmdZ2U0P+M~zL1FCB%~L|dDKWH7AOi}xOM&iZATM3j;^M>*tI z$^ba0F4|ho&TDNwDk^>`26I(Kh#M8*3T!i{52k7B!fpE>a8dbh$Qb_r^v<>m%1l22 zDcp&196^nQE{>K7f+aU+?tiHniu zUe+ErX#jt2{)VVPsiP+^`nEz^XpI07b?gDcT{p8A*RLf>YUR6NtYn*Z+}ag1+!7rj zkJ)0hG>iQe7p$cmIuw8tez}x~Yz8^MX`KKV;cvsbWh9({){A-KitIGCAHW_Yxb4ct z+*`>;tX<5FIbUM8truY3isU5M#7nl3@7x2JDVEXiW_13~e}(VNR_Tuo3WL2}L9W<= zJ7s=2{in=c@JaS4ZUZme&2R?qrb#BF)eE=(C)w6@oK!;u<{Q|>4ZVs{8HLI;_T`1Wd@q)Z&km6FoL`hwcLtV-xgu9Rrnj#$xT`IKT5(1CmctUqO&5<#3DNg|mZ;%4ivZxqv z1W=cJdrsqnJC^OboL{6sMn#u$@#aYhRF8*!ep{TlRQTBl=ZV1xb;!?Uli7}RawI<2 zrQ8~23>dpf6w!>Puqor1YTW^MHz!;*6dJI_Y)qU7CYoHAj@wdHo<5*6aJdvfKzt6Y zK~QX+(gb~Xt$V53A}&M4SdrtZ7Xw#(`F;*1sSfu9+2&v6R=-jWMcVja4cB66nD|q+ ze(CoA;k)m@cZAtSnZ!E0)n-&*?fl)}+t#IITDj3dn~+;&e{o56hf&)p6_EJYWyk8# z*V@44vX3Rp@e8yK<~vCwlmzJvQnjH{4z5L+W3}eP`A*JS#~BsT#cbqqfnYts36=Z;?=YK)RyS-hOstsFd@1=(THNX^SBTP{Fg z`$6&>cTVd>SNFf!x}|xkv!txP>^tHSax3<$iipa9o|D#A@}0~cxVf(?Wx`rC_Ru?Q zJ-^CSL#lnn_u43}xUeM5B{rkR|2bk?mwktEPJ_(b)zDt#B-<%#LR=UBY%e-!v$zQl zyjX|=halD#gD(a?M1-$!zZ!b6Om2@;eK?;mq||yxQ&W4UtoEX>>T2iyciL$Dma4+W zqR1ig^^KR-B`;uDvPrJL=zvQMjDvSP%)Vcvn5;_6zeZ}OtJa9d7T9^s7afUPxOTpn zYlqJ~v8LJqAt`(2B(f?`q}stBpo|yy5UFIA%h$(cz1+FeqdWQc;TyUE5|-iIa(mSn zkxWMaN-J4aBPE|$3y8~FPLos-Tw+VN3IRxT$-5DZx~#P{N*jzK9JSXGK@a5GZ|bB^ozEH519J?rYq@BxzOVgX{FSw%5&6Zk&$AJ!J)Q#@Z$MC&qLDeS(zYDK1WA11 zI%QsPuz$(AWey1Cget6;ZJpvaBSRXJcUwOyV5}oS)=H_c%k3T4%A_G3R^Sd>FGFH# z+-eI<+NiozUe3)@0x_+)WdwP)RlOrjGSe#A+;+YT^Z&SQ?ZmiNI}g3n?5a-``H%ZH za5V_~Jh6SQ24;wCK4(K$12eQ<@^z4L)o>JE*w32hXBm;|rgo0&MlC+y578XGni%4T zx{XDTo*{kwYMvkRX2hwkUbS=zX^mX+dNz@u=!hq$@zrvNJgvA+lAKabQ3V=VHw0)A zJGYtdq`{GGtz>+ytBE_~Tn5`$lUG1NLCk*+-o-m0NFVD|Y3Zut%zyYNTMppmb7P$t z;v9!mw^Y(mD++-at(S10;4+HmS3Yqy1wB^^42Q61Sj$fe{)fu3dOCx4cq95FWL3j9 zOQoqj@3XC{7%Yu#M=>G!wvo>+f(Q$4c%H?dwBXQ&{y%8Pj(u{DJZC;)aV@FBftx^ZqycgUx zv2X2YavzSUi`SA+=?<*m#7hCVjyzIC#29fnnLR+#7KAnUn60j?~@r(|R zV#lq2&GBUq_xI@V$tT}yYgtWUm9Qo*(=s_Q3TVv*O>%GGZ_a0XBDJC&X86=z1>TxYb5;6rNR ?bfJc}Ghk}Crl`5e`=BbiwB zZK5>(4mt8h<9|=KyGAsGqQTkFW>&(N0m(|Mj0QeQ_vGtLQaejPebJ{a#uKsqWDB_z zqw(Co%N0oE_4EZeX`Ps-`h(WZuMu8XszLH(XUme4AGLn8g%3XIqhCraoIT{4fVHZ& zTgkhmVEwNCE-5!=o#Stw}2PBpX_IsENLE*ms>!3N@q zV+VDE4c^G`6V1qgnOb%;mMeH@myFp!zEe06_cMly4BiZaj2uKlmz$0W5C<1E z4c`o+T;xkVY$n9#atptiyZfC)JgD8w0m;e`R2eOOD|0nlTuQgGn@LF(5n$S*{*Pxd zF^p0+Dy~~S0N{k4mSk@tx{>!Xc`|R7nLz@rKv9kIVjg>CVjozu)ZXPu_*ywOISqHM zeKXkBXE~1-OF{0Kkf){iL@pJ$_NYX7Os@9U$r;cU0r`QOG>IoXYu$t%Coatj@*1+F zmk}XFCZe6Jm8@TNMNF9ewsO-q7glp?rK6^DFt3+6!*!ghOa}SfZU#5QL~K@{72h;O z%v)s*_7K*2JIAWr3}_qkN$$>sU%FM+_$EBkz^&Zv@G0;>gST=eACP~@Er)Jy&W5uN zqv0Fal2NgnQJX;Vm+qZgw2xzei zzLofc^i5&cCrjTXDm`kAY*~XA*1iOBZC3CcWkjtQ0&%Z7RonULIC%li*kVgPWC+qSDRWRa-^m4BhC7l>_HMNHaLZdsWReB0-E zmgEVpVfl9KbR8ux+R(Rsc^mNLRaI<-sVa~Dk$mT=sF<%?x`Kb(2SMy68;zgSoYiCB zc4T^7&Sv9T%Oz*jlCGs*dgUM`183IAp- z?TW~wK!js%e><@XUu|6;7pvw)Xd+PcQZ*C>Tn&HlyV@R*vmNx zc8WCCiU=rQ8G<)tX<5L^IYF~oFOYDg5MrD_g$f8L!Jd=w-(} z()My)e>rgjojmz!QLrTBZ?|^U--~#pisfMQzwF2?)7hoyM%M!J*2e=~?wztu6ykPk z-7=aq>Lf)Zv#jjVz5L?jCO9CCmdj~B=18e%IYCxlPSjSGdJl%X77&s)eEsD-c6Ik8 zp?w0Z^Gx!wbEvmAOJk27;I6w>8q=A9@E0Xp%msT2m3uCZtg+9Sy^@U|ZnzKVD?Y35 z_OSGWulVxLb6I;Ow-SljreWEKUnz7dLI=uzY~&T6Z$iSs*S!*mgp3JNGn!Ax^B{y{ zQCCz8RxG?F$Fqc{scZFYT?`}U*RW2U^ie_r79TPlD!-BeI!f^23jl2B8kQ`+D*dda$3 zUL~cCgdeb8nc(S`1V8)yQgfZpf@6OZen40fKup-$^Vg-?bay>)2{C8E-)VpWivLahi4# zW%bqE4pV#5(SM?Vg53lX^vSH`>|)aGMhvEfC4RGfW==)$h%0QZRYz^2z#=qi-vJ7m z&4~!dwphK3#Dbc_r2zDmXIYx%@*S#=jL+lTcH;XJ2k;p0=Jzg`AsrxTIjvVs&kZK4 zznaK;htXiEq;Xiua&qTAV{2JXQ+5XE0v?rHuO<#x(d^F^)wH8K!)D9@41sNBlTZi- zw;hEeI($0o?_>!%Q`|DUovRDCtOzluC8WRYNHGQbciUI&o_p^3GaI^{tCN9>pa-`d z3$)kJJdWIUtl}K9TKO>Dn%-@r*@cuar&%?nr5!7c$;HMKmyAap#;r4^o=?bGIEN>0 zJ0bel{AZID&=41pM|Loc=>Ex9{gpLu`#TPV_drXOCh5WGiuv4vVs0jztC{U_;e6z8=0= zuq?iv2Ne$baAw@~$^;U9hTA~Zz%40hvVq`hql?Yk#dl*`>3no*ZIv11s&K}(Vgj^q38N$Cno&v7}pr@uaHFC*Ct?T#=n#2kYqntmejwK zb}pRjn7;4I45JC4a%i2f$?x3$-~PJ>E7m9j;;M!pd4S7l2+AZRDaP=q`I0T4wRTnK zC_L?>;KT1t2%h9y0 zl%}ae%+8#@;K_n^tpDkBIc*7b=|<@fj{mf6W?fdRd<$x;3}B@L}~tj2L_wW zB4vx5gQzvCYAmGY0dB0BQ0rJcZLRP6$|!&sTD$TIf_~B#v$D&@=u@Etm%dx@Rm4Hz zRwPhVsG#LCo*6k0A}Gj`LD;1w*NJ$QF1uB`kRZ05gyOvdsGJ3p!1w>81wYw*DSNh( za1wzO|P@?_b}n#Wg_PJXK_=Hqj=T|sqGf;P8^+R4(G z5g@67K>d%tyWTBZhk>`vSMW65rw{F zwP>12z5=H;nokN(cgDu@3CC2n-+0yz5mKB2_2>*BuG&PF7EOE3ChsauFinofoyVlCm9_zvB;Sp=ccYJ~W+7{Q6;6|ne`57-oW3DX3y(pVzB|yH;T4MMt z(pH-9#_as%Us`W1&#HO2v3~X%(j(j^*-I;HxtD_2u>sZs4uZKNI#{c&zuf5AXdBst z@Jlv{iW#_B3X!1JO$)ZF(ZRB@XJxFm9UwjU4%;apy#=1zzE^hJ6_>AI$7uuK%Z-0u z%?7`hm@o}jZ$nv{rMykVClMtZj?zltZX@w~O1HCGmW7}616P5!Mx!-9P`ERcw%u8E z{CkNz%7%&_|9d`hGqclpQ%qz%-HXuaeJ^KQ)_D-%MifOvi{P|b1_Z;SaQ;fT(<%e{ zzlao;-!Y2ftn$CM_&rDF2i|T=70~CL;TCtcT$-0rT_+2O+-mP|WwvfKuZWA*%kRkG zQHzL?J!}0k&N>1#Tgegx4FLernwz#3V~3;nOTc>ch=K!NVWW&zni_6ln|C148bBWj z+g#?Aom{qBdf`#MoByo%Xftw58)*3+U%(iHo_O-{rKU{DzQJ+o)z5-+bFF zt%BH+ghDoLE8dAyuFUne6RvVv2`-?6mDq`ONmV~W;~&HtV zJ&(ko{)5CFBFyeZyg!lk!_ZiotaCP5wHa~g2+?JYJMWOM#5y$NU5NwWq_dSn;{a`E z$6ajp&>ZBG#89VgsQ^Vlfg85`gS(enL_rCz&VlNk?A8GT&q$nO-I%g@LPh={3GMsl zte+(#nVPQJnCJ#D_aPNuD-D5L@=KAO;(7w1u8<`BAdf7!qxHdTmP+U_b>kHCvh{;J zQKryp>t45AUd$~V^22NA2YDhXJSE<3`^z)~SX`el9RjqSjlagJobr(fOIiSMF1tKyH46>X=z>v zL$uhc_j8^N^fcseJKk~SQ}PUiHgNAd70k-(Y(Gl)c5$;op)~NL!cWL?Tg!aj27iQTDW3Qg(=CgNu;kFwt~`roCnD6DO8vP@x*N1;FdJ_4>RHX*zKyhBS?gI;t9@NcAEAR-v&Ar>mX+xBuC2X&pv63=xQEdDlB4GC9K40;1K>2m^U z#!9LBl#Tx+XJ>i_`xOKiIXnUjOk@M1es6d<#+3tVH-d3jXHa1fxGZ#Dd3({@VDF< zx5_)+UJ}afXcly4)@&!Av}1{5w(jV1aH{!)4Xg*cNX{bpiMZ#%_)c^z%+=Q8c%c;( zK*L#Dbic%m8(DYo4$tnv2(6aIyC^C+SJrb)J;LHX8(Vjjeb+nhxAEwMoY6!*DZimT z6H(;JSKVWi*_k`wuyZ|cdj+msiR0LBuIHNVnN`S#R_WBE8)LGvUKlXh!YMh@Vi^<4 z)N{6Ur%BOEjMs1t`;&17N_Wji%7D6sI|i1~br8?J7~0~$~S9$Nu7GLt!wX}gSbo@*{10z1** zd;h}hr#T^q31mT}lb`x3>^?XfN!bQ}>I=pzBp(^T0)K}phtP@3volEE40?NEX5cjfp1g_Q=z_7ePpm^SnquevfV3jHt7H_Sz zf+HbO%YW*7U~12tZRC8#oi@(QxZ7>U+|LmX!?p^P>8g1eUSd1niOsuAw>oEc3Md|s zOPOkkarCoTCDOm*CaYWBO&j=`W5?Wlb5<64gFo{{PpiS9tiX}X=@C?VINyo=!~%hs zftPILXNfjkd1|E&mR&5nNP~?=tAKW$r?D)dx;i109&h0loBvnkK0H*F*P38ET41_=-X&M zVHOB-jQu>PiXE)@Y2)Q9J_Hx6{`1@a`+ry0wA!(Ws4BX~jY|`j1 zM77Hm2+9~3+N93UeP7JVxwM`nHUZuA@3w)WSHwi#`|;x0S*4oBN{{7g&RY-bteBbBzD3TzViK&7uh zNbMK7Vh5+?FKq+ANZgoFLaxDIBo(1EIi-*a)`ouJ81?2SZ1@*GG>e!tWo!6c1I1+Y zz1n&5udMNloM6~Kg%Y?6XwTxnU7+=gL=fzWoFjug{35B1Yz`NHk<{k!!3Zs}RN$G~ zjXUvjfd_Ff?tCtQQWQ>9;^|Ey2s*js*zm(oTYb&Ma-LP3}r6Z|mZM_Ud zGSS_(ku4P;T+b9;!6MhfV2-w8O&)%`ZRc0_zXwu@zjjI&gr9ue?3YQv>qr@X>8tI8 znQZgFEW5~sQu5pyio%U4BJB_VG6{v-amCRy@=HgCfB6fmWmDKKp!yh%dPI%Go%ENv zJg9s*TE|QOQ%$R9r7pKq+zX?zSHH$4vn$OVwvKLq(kS&Fc#AcEnPhebS8z`=()wjN zJFvQKXS15N`3`d0ewjNI zt-fgD_AmVifk(HUs%xFk;+CH?`;|`w?Xr9WzseQ$Q1uE1V}rjcmX8E}2O5zcr(jO? zkU9R!nw3)U(IOE$!LEr?UZxz_EN1zjClUCkQFWlPHGh?O3_sV3%5m9ZYnOtDr^vaw7*CA# z3e4?NHq7yf&vwL`TP{OWP?=5^a60o!B);hu=-qDm6b;?7-mj8m99%fyfk*mT2rdWV z$Z;gN#I^lZB5Jeo1lM}FuC?e3jAHYz#~5Z&+S^7lu3BnDb(WKID`rM=w# zRpLYh9JifnP#_nGAkN!<9f&ZykIUo0uYIzhy9aFW*Rgh51S!)!z3i`ZsbSI|wc&W8 zp$p>ps`4^@ii>A08doBnIU6kxnE%>Xmd@<}0Vj<4uY<*#vih&f72R*(Ec>s0B$Wtf zvJ{Ab(R-{>UcfDS-pyw8#+0!?TTva$W101`^1n`mN0xxSl#KBgb1-Z?Ou&|YT^1zl zD{(gbwIlqaTKY1OS&uOmk;;vvWfV>=?wju+$e_(EFxlHCb5z5qv_?! z(A>xsnwQu^viVj(#2qs7s)`D>OQ}8BXzW&Wi^UCJM{(Ip8+j#nk)#wDu5JaFmje_s z%92vFv*BpjB=%iaV1|54oQG$MD^RbDKAk67QaGmqGIzSE7Q#97~oPTz({gZ(B3Nh5sb z0#@mK2YwT)E`FV$2dhU};lbbd?7&sq&~I|DA0SMh4gV&|=!5to#upIf7I<3wO(Lfv zvg2SH1#v=sqmBJ02TY_1L>D$*iXNVpV?#Xw@NV!)M2Lc!!z>WAhwHI|j z_O4T#9HFHPutwVPb`(U{W{c(1JVb(RyHt9IgW_OV{!P*tj7YfWvd#&PB+znW5XN@{ zoPYiR{9`XW?t(5Ct$!ErAr(4pE5AwZ;vd;mYgs@Cgy{SYZWilhPWKR4-ZlyZaB(h6 z+blyv_j<{;N`G93pSSIRWH(nH+bPYw88+1Hw>itAb@;dq{MMm>OF%$5_}f@y{1sg6 zgvFo@{Wb`Ju4XUU@Ne^?2sT}YmdpG~fj6M&QP+*k6(v?KF16Ag+?}jXufcMUmUmpd z6q@`R$KF^~cM5a7Y4{8d_uo2l5$xCMzx6%!DCvlPo9M$5Ad8(WZPC7BjVsejqZAR> zjho?b9TlhCB>lHZ1)R>yP_td$Oow$2OnVGhSvJJjMbH2IFSeZR>?K!xC(20khLujY zbO`YUr+QK4Z8)682;_CSl7Pk~G1%O-d?&Dp_R5kItGcwjA7@9~$f;*jL&G-9x3`Fi z+zNOailDz;M!~r7w$z&cHVNXRWYLV^ME1MHV6?+hPujrmVzJ$VR|O1_hUScI{}%CxMGmQX3$*GoGD@|{AMQ-%WXL$JqK4)jyD6^(I@ z>0Bc&6U1u!uEx-c;+a%l;3WuJbGrY&kn3skMh*PlSN+d1D~`N{92(#*AkPnY_vxqtC3= ze_!xK?v-45{0<@BW+M8TC5Y@~Ksq;Vjcfweku%EKV{86CSJ)**xExi0+(5@Hqai|s zWH+%OfsUc1Tg=k3PSTZ%6y3J;``~+!7u$05{qA>JCrid%+~Z$Zx9Urlqk`%w>y>wY zA&RID{py{pPJb=MY71LbfY{zIbHt~t1-k$8g6D_>=;l=qvlOhK?FDeoAi)Nxmkl#czur=6Sr z1sg9Ah^OHUk+X9B54q69WKeE{i7cZG*eATq3pSZ;Jc|QqBcTxdI3)iegUR|m)2(7y zZq7HSPS0EG4~h2g;)rSgAvq#1>{1%$N+|^wv(vbg1nFC=B{3-G;z ze+)!%A5!1G(2sbhsBE~zMQE?#KgPL&Oc+`|^2Z$8jLvtIgzR9sIBje=8pHS zcg7ENXZ+xYW5;r$Bub(rO5!9MqImSc5A?tf^uV(SFd)Ey00ROH+}Hba&iUHe&fM+X zw0QLUd_RBA=im99|3~-#Y3ub)QjBK<)m%LG=@jT`xPp5M(t0Fo$ZU)AAo=#_|0stR z-|Jm=G-^M=)AoXmRb!HmqO1)eETV?g7t=PL_0g;#{{J7<%S31gbXvS4-Z=rDcH2y> zIu~&6f0V0;^lCoFa0;Vne{8`UZR;i9^rHX0M>$cKd_nMbhjL#|u#uOnL??u*V)fi! z%)SvDTLMdhk>Dcgz7YtEc)(6LexvMTVUIa*4c~|xw3+9i@f*J5XJ!=uw<&%pnhD-( z*^x{wbW~fuQ5Nxla8GND>~6PyG1`2f*&1Oc5QDX6`;sS-b?Wf%{ku^|_a3y)1Y`%_ zdcAK>#hW&eA3lv}WY|Ao8HS;Gz=x|1G{fjO%KEmGoH7R<96E@sYAmbGDuH$_!6-53 z8EQO04@na|0KNs@7dqwXYDvQm3mcr7Z^Y4uqs6k1ZJw)YdHzHNs+sp;AxREkEwJzn z$4!BOI7wpt^q4N@^tPfDTB?ANB2|>y`KFa$79e)-&zC8!^oWUK7iHAKM$E(DY^4GI78Y`@aEUF|c?e(GFIx0t#_^!Jy zn*?bQh8N?25>?(t7Gvqj1Wts%-Mu!t80WidhrIRiFG$co{z)72CE7E%(`r5ooK_@K z7>|KYqoFwsYLS=AWR#V+4&daWJRNhFRuuhi{L-vX+FblnT#ILH-haVU$w_M=S7fk9 ziJ2DjC)qO4Z7w;8F3wGGiKo1heeA*y)9jmm89`XQqjlfR*S#bPDYi=W-;5pGiW72t zFIk)daW#H3F!HA%R0%-Y^vzhxt)c_Hv0^e=i+_Lxm0~7a6(JX@nQxYJwjImt_6i7@ z82O%deABn&zowCqbvoc+SJ~P3MjK%f>;<&*Ht@}SONuxCaKa^-)yOx)YpnzXC@B;r z-O1@e8It{qUN9<;GENIZ6DMBBd{+A6z5`&wf+GGm?yaAd<755U4?$>VxfZPH3B60p<0YhJcUi%U7OiHP3R;o_U!q;o>aV`5+;`MM6rmXH;v4r?m;<8`=t#TAlT5##V6<0#e{3RXb zw_@Knk?^O-u5XMU2Du_0=RJA2+5za6J#+x`R4kWJq%SGpHA zyxHp^O2bIVTB@ENhaW83A#441kT_V1fGlLJ-j>~w>Ks(k{_S%3_B={@rw;EAfy;g{ zT4(kLLRVTV*?;tYJC8_}3@lK|K-QM8E%!lgmBZhT!z^xF8tRc8;4{K5qq!<_qBt6J zK7&Q&OE#7*(PowuzqM?+zX#i+@oxumRCFZrOigA_P22z0rUMdTQwjf{gL0UOGQlP2 zh^}RG-pKFZe6C6tP8GNj`XVF0v_)San@dT`)bp6tG|?64i@Iz@m5+xRrUf7UUP%7nE=ncipog%BnZk0Q$Ca3PZAO)QN%9BOohpyAG_AWIB4 zb09SIB|pDHi)U;gex1g)iye0ul&^Lq`<785#NczR(x_Kf(hvxyqgCg!&UTxIvFJ`J zzGtkKLjlY%$^7ssK?-{;LGGW#4nz2^S^$aph8wH9WbH)-0NRCut}tSPOT`j!NHL z_npAi(9O7;e-qd?)VJfkB+<^#)|W z-O@U@cvYnRpS0HRlsRqY-g})c7rM3kFIEmsgLQl-@YClZcsjlLY>mL)z2EVp>X9E@ zSTj&nhFBW@PRu^uMbvotS-H4JqrJ#w5fINog&zHm6AACIYqqgm4A`hJYpZ2Xdoec{ zuX<91h2kD!lhq{N3Go3W2w@gE44IimL6yyj9}xZt8L$^_3V9CesB+SJj&V_fQ(G&PZzJ}k8E1^Uz&_lf0W6?;AsovAG z$nR_4jVE!(e&u)__wlxFw#o07TQXbwWV22C>J1>hok=j-L^01*V7MKoZ0+U)4A?fP z(*G9=NYu=N!MBbHZ{G0ibvGB{Bxt_X2Kwub5ih_ws}g&;c}E zoXWGxPK8{xhVS|FJV>mg#_#z-;S3@+D9S7LMxxuM=*SHt_?IWIIa(r}vX=NRPwh9L z4NtJtt=}t~ggbEC_sY)p7tf6L@0Fe9<&)D$hj;SU|BXvk=l9|W?kkxgaU4tVAq^nK z%gPNzSq8;cq_g2DJ5wGJPZ2xvy>hhmZjJh%A0r0dQ6CN~*IpY7_(6_V%aOP??ooBZ z#=n=+o1#UQRAVyNj?l*J^K`V5_*=>lkybM~34*dfTOb4W$%{H7E|E`Oe%xU8g;;No z!cb67toNeV!w#OT;I@==Ml7mzI1PQc?AY!3_m+a>f}T{~68YZvt~W0E@sZFM%Mry=u%@MQUZB7GtU12Rs#xB3 z$~NB;Wx7qW&N>C+Y>n}-li(F?iyEh97233z@mu>+taY(%ESROE0upJkb>Kx$kbMH9Uc}c?K@G>einEK46Z0C0+C`D6^+bX->cin#E7P{4KFitK{eNj= z`S-{F+-ghZOn-i}jeF@^teH(#qfZxq5S_{C1dtg)*v&*0m#|AO`-08+4jmwJ`Mm!* zTl){TP%YMT5&(HOT{|P_Ktf)kyn^!4Z7k>LusWwf`H#Z{t>khcXT+hI)!|9&k5G&Gb!}kLf6MoPd^M?X&HGMxeN^bT=Rb|cJ_tS)3KLlFK z_sjVh)cJ0Wu0l34uoZZ(d9|f*idlyrp3w28oYOvTnr|82i5O@Pk$hsHEC{ z(Z+KWmPzhPxF6eOlu5dvm>tsCPhTq&oi0N%o2f=6XN+dVI?a7Q4)7Oy32bTeRrk^* zx*}`g``7C?JO?e;)AgChGZHNn7xqaTtw6w9w)FCp9gQ-HH|4|f z2KaC+d+y%{PVceW4$eVg=Ov&6K9Tc3_^lDiV3pc>-M;$*4A zJeOTw%MF1SX;j1mpcnmb!eYh3l(&I7onl6rH#s%7pB66XZDD7-i+~a5}`ZH2ts~ zGLdjtSTt8XDOO4$W3A>m#7N16D;xF%cSO0Xi#VTu-s znM(ecvl^0FZZMF=(j!-CKrLe=bD65<$yv*@!SCxI<_IR_Qqa`}fr;tv$} zL}s^=(9)gI%o`j@yBw$hiwUK=6PByHoIuKSO}_HVut$IxtG^ukM^LCiy@n0ZTw;uL z25ZbNwS3Hu$Zpnjx$MPS4Ee z)CVA?lU~51Q9~MiPZmCDV^Jm{k&Gq?m20)jai9pe3xPJClP%oGWHpRTR@||0I?6b+ zMZ_b^t(nW$zZc^6cyyJ9E-v_74x)fzJ8Zt{LO%HT!*kykVtHf`Hg1cT%T>Yhv(J{Y z@eb*=Y}r3`i_2JvjTb`1wzWhkEO43q$d5_6c|aj2mbxf&6}%ozED@@D#Y`UC0eI7@ zY`RD)WVYP+qXJMd2y6OL94c5eo_R3GkDLxUg`fy0gX$Nj(^|8#5~j)0pe=gWErJ1s zz$H7WH^Kc7`G9r!Sdi*H*7>7++C<4s!uSq7=j&F8+HvD%LUH4Kv4V>|{W-3D^t$((}1 zP1tlamQ9Eb4w+JIChAKnIBs(lKv=EEl;(dFXR&`5Op7fze4G1iv4X|PIlWCw*&P=I zQZylh4YrkBbMj4Ng@>`ykIQZT*q=39-H&~*PQ$>SvictfHj$74v~^P3|Km7&Ten%` zkINE$VqOhlSh1;4*&v0M(tk^ zo{LHA@Zok56#vJ8l+h?kF4kN1N-!8s%~LGsKuksP)K1y3xBbdN8_BkVT_ptfXtdSS zr+AyvDjxlDEcI@zCa_HUacnGcRIK)6KXccK_hN&_y&BG_w3&?I&+vwmzAQV$K8zDN z6Jsi@k4*nx5H;ZtZfmpo3ZiK#?0^drB3tONgw$s-R~M}~nhHFTmK*}f`+IHKxApT7 zKP%B%k_O>$*#+58a&tWahJxzU{Uq;G6;++_pMxi=|4E?vUBWC4(H1gV>y1w(a>}f+ zLzF;M_FjkV$Hit=-IZPlr@KsdtsloCtSs*RTxwP zA%$(~$WcFYfO&TQByf*N7NL{gYDlst#5N85q@3WM{h~Lsfs9#oVn?z8%%_yu#PzY! zgzRD5J~32xkaz5`F^5rp(TFOdrCS&)iyDIKtkA|aoAhxxumr=4${yQTblEF5<-c-dKNA9{d7OE2Zp3@BvjvX8z~O6rwr zl1~;gjX>%~6xACMOKidIWI48at8!)5FUPigrkMCPEXS!qaUj-e0_);NZ~CbRttq}1 zZh;*tVf3TSZS6#8A&l@LXnz9a*t(puzlwKlTaF8kh18QG7=4Jp`J8nmbRTd2*e9)X zxttP&K2Y<$*$AX2vx|Wkg+MH5(}tJ*YV4+%#zpvL;a{LFSs<)q(?d=*; zzy+)OX{M2CvN_dyQ>q{HgE0JHNZ^xhcz7QISm;wpsH}3n)9g z>7m7YAht;J$3AYYIi3=FxJgo5bRgc{T_)bJcJDxzu-4&!&({9kzq8JtmeW{iqZ8Ck z7`+bsG%i1^9m(`CPyK0J9BCo%zsX zf`kKye&~afwv-cLQ8(CfRRdq&&_S;JwA|;u+aXyF|qR3yNhTNZ8eilo<7qhhkdbC@81nj3t zit@8UXdKv&%RvlAbY)U1qN{GK2ES!}NYXM!Kb@^^PXN{1e+Mb9`iVH-5=_{|^xYw7 zIx5S-8%a_Q$J$AYEBn~&0p+0V-!_Q=$#?&oENd%Ak9{^!1d(ocvteHNOw;pe`PMIzGZ?PP6Y zO;K4UQ^j|e_o(^jac+h%LbMkDh2~4y%Ub;x*Znj|tSws0vU{A9C3EU_9}34eP*X?M zJt<_%Q8{6qKabt$-%(5idSeecTpurb5(c9v0)PT)if3$>&;h8Md6=$h*N-et!L)OTVBX z{dkU?1=Kp0*MHTj30CG|Cl){%B4>0i~ecuat8P zC83bRwdhp#*U!A=nNr8SMtP$C%qDY5ctj4cyU=#i(S-t6ylOMmucr~Q%|(Y=dbt@X z4flN1kluX47NSPM?m`YM$YR#uX(2A9|KhywgmqkwZ^}G#(pKV^4*prQU-)EXm`5^$ zUnISIV1F^ss{chCYmW{r!-8MrlMNfB=NZh@_>0($2MOWS6cwZl(EF{By3H{PssZ3{ zi5e10N&ur+maS1k!3s`V+b`nCJc_LZ!N9!#1G^MFs3ZGA;_<9?#xGt%&QWi|kLNlT ze7F<-qMXbR6M=pB7v+e58N-5V3ZxR#3N!Q3U*y%emNLwR+DCs82Ypk!jpan~V6<8c zEen`eY&=26{uBf_>8-lRy>8P{Q4H@HoAF<`=3>ihbKVRMZ3%ERpA+C_7X{UVk0^sf zE_wpa*2!u64N&k>lquxG^DO2`NqwsYR?I@t^QUaZSK_N*U1#>o*e>K@R`*Lk0Fa*a zR{zUD@@V>$pa*rbHT*IVYcF=1oPfq?AxRuk-T0*^k&X6Wdxmh|NB$qZ+$h_Z;kmK0io2>vVd>;RXJaMa1^}r&A$o+1HD{AcC`3s z$hkps#kKw_P9~Xotu1;J3M36*2`k7xU==g+&Z>#hBFUx~|I%N@ z@!fim#7MXtI>0F7$WiRw5xc=geif%{M-S${HtKNkJbsPP*GKas0XMsL+SsoWSs)r? zYw@UOx3C?vS^@>-qSRd6N*j+d7jF2ioy>^}S4{sZt^|Ib`-|3lCcD8pUk)g5a<%4i zZX93*?!h(4=5zJ-AK+zOsA_nY`Ztj^Eavw7#b4S|1&9(LmAve>>^ z{d@mT%phB$H;z89V5|QsPv|GCEl1sYz}o!_T3$)&I#$YoAb5s#dZi*R>W#s)+KTKC z6>S46<#6@mB7mSK`yV6~ppC3tzoCLn8>(QnqZL%oLuVh2dSw%JDL}){3yiJ2-Hf;+ z&EIGLjj=VYRso9klfFv!(`w9VsLjc0-8_=I?3|~4);oKNK zdk>aG$O}0T#=>&#mRsIp)*)P$7ISGO4kGdn52m{81Icb~#g`BD9casbo!D|pQ4)CS zl{!yx=<0r5u=!WItp3;KctFD9^k@ygE(hnW!(iV={{rzVTCVBW@#aK?J58`T%0%MG ziBNV(El~#dECvL}T9RC385rqt$|M48%QBDEE1MQ;|8*dit=zYc1ne^hc+NTl7)eO* zJd9ro_`#9+bs~j5vI!slbvbOGBZ1tA|M^+GxkqC>w}AKTvPYw=LQcGtV>#=$N5I*& zSV0#)7OEDEDdop_w3VRw&EnCAhnWxeAP@GmPsRP+XM9zO+r{gV^Zq*VbUB=2OXg#g z>&#KW?-H*rR8Z?!$(;E$Tl{sL8O05OI9d92l7svEcj-Z2&XpVJl{HdK2&pqe*>Cb; z$6#c)A|KZMrffNZ{4hoOjpGXG_+PVz_$6|Mq!+)5Vad@T8B)`4;yGed{T`mM=HJ8y zV$EPJzlm+xRLqlFqc_p8=!bn&UjA-tuUd6MQ&b?FWtBS<1S0D_X}vxqXZ1h{uOso_ zKu!pj&aM83y$#ywBHHy|a&5!1XEb4Y5MT5zJDMNAD07stoFP&-vDLM#Tm+Oh?jXa2 zkPb260u^8nGo9VE6FSIdVqpqB#A#Q)lDViaq5m11|4lieAO43wvW1*`Ka!Hg><2y& zMR5KqIVP5)SIATDvWQfegamNco?+I}ep_}A5x4we>ikzxaU2l5 zSzC;wv84l*47S~?bERH~%jx)SoXrvqM5_Ew?|{H=h!c9>0hFw*6v$FS-htmZyv%0Ik`65(G z`asyT`m1?~q2C&=mX-V@`4}3bZdwW|)nHSW6*f|9hy*Fi!Z4zQ8jUiUMPo;TaOyQ{ zz3LoAyS2qv$FefFLKb{Ar`tp57#Paz0m2HGd5{hn(XtNCQpJBQ>J&K2%{yxYQ6~CS z$qXS}Y}jYmrKIXM5*4t!=llz~!+2Z`JSYasj^;4Cx(Yoy=3j}5N<>hrMO`tNq`#`b zRH}MsyeBKLw7u>`Je^<_V>}r-owAuI6M-)$2hqiIzW#soM}K7V2}XAh0Vg0N6G-Mn z4ASgSLF+h`MIC%KE+}kE36y`(mam3E6bO&psBUw;a`k$`s@O4xv6}rZkUVcTYID(P zR`1D3e7KTrZ?Eii6+za?2X~ zYu56+xHpKB2e4p|j}kh*W^K`2#-cFNhdUfo@@Wa9QRX(amqYI)P6j_B# z3da&EF@J$;Yrpe_76n;i+H&c9tHymL59}bH((lT>`0N&&PACZ3K!B56m)%8UK9~JN zXr)eTJ|?DI&m*=FHFO;^p}=AVsmE0n`LUE8_w1AJ!LrY`hgTjQa@1F7+ouY8B*EtI z{ZL3?pb!^t09@Vg%X)89kd^x1#|FwRbkG|76H)}E64A6gJu;wxS!w!xnQu2a4Sruv z*1#YM^nUM00YLJa{+3#bVOdhoP zfLSJ&kdD81`yHaK%&6}kVx6;|Bx@ahtEME8MTyeTS>$uVQKtB+3v0(BH8_+ z93PGxWW^ui_$Zz@n#5=+(V46~PC%cu{vlSB=x{u)*5-BmPAYu7&cq*LkCCbODd=BE z0?G>_shN&`XO2}O8(8ljd=J6B5a$xEpDt&9!Ul3gQnWuJ3gZvu#6b)`-fttmAXGhz z2w*hE5iOzg3v%Z=n#~co6MVwPa$F7sba$;9SJK#`VNJY{6$cSS;SL>ryNn-215%@D z?^}A+;vF^BzCN!**FjHjpAzbeSXdM_h33{i4-yC*<%)GLN| zES8Ad4R-EkMnZMQ+*+#6u~K4&udy8)?c6FVJfiQdz7is6xp159+-+qxJ4HGc%c3?r zCrgd5{-tg4W^CK5)*js!?OTu*+j1y&9!Fl5j68A`d_`9bwfB;3&oU@jA=Q@&;L#ai z=+vCFZlA$y?K5~+_eOOw%UVoqTCFe2*qzFXY9PDCbxZFa@)sMjbMG$mDSM;q6T6^0 z;sJs(M!_LJA$UU-B7G!!lg*Swb-eI)I2Pc)BODox?;f&1aJ*_qe5pSCCwA1%$?N7n zLj)YlCG8UNRD4>-@^}`zA(#$39=#RA)d=%$qA zT-avw{9sUi#^$0%;pWOS`-+{QWvgzu9*2mAe#PpdOp-u3zD&ZntiGY#J%n&4 ztp0{_O3-52n&_BeM6tT=wb0THUg;|w-nH2uBVr&~X!ph}bA2iDlNW5=4F$V#KAwk$ zXu6?X;H^7teZt5h6EYnk-wW1!LqUxFJ6-j-;fA;{>`meRTC$Do0_jn%MjLM^2!?<| zY))%b5bp7?e~;Vdtbn@O+MBd>;hI5_e5dK>wvH$<-*^sZU zLS`vb%J2%FSZ1YjzBDLa?0kIo=r(RGHeZ!Zi0R|TJZ~3bV4dY>Y$10DHzd~UVtf=K z_RqE$&Gqy=&xle*U-C(OzKGP9e0YLY+U2M#uOPkRQxFWxH^g%)+eUv^u6T1qiCtvn z$-(#X>~!1J{E6I>(_uF{O7S$6&aSPxF|OVolu~s!#y!VK<4Ie6V~{xe=!^(%(Ha?2 zA&)UPwlz1#?Zz90aF(!0H&*X`@RYXpMn_Qp9c0y*T|#^!;9i$M_lb(KrWhWUm{)y$ zj0ds=T?!k#WX%afp@aM}+HhlBWyL*Zo)FqT;%sgEM%(Cr!vPW=-uqWvGkD?63A?Mt zu>^_X6}0<#+fohxTnUFlWE^YH;R`7$74}oM?Z$$0KZcaBBR=y4f)2@?j0|9VHi15n zZd@iaoemQ$NS}2l^bslz@+1cqwr1Y)WbE@6dv_{${Ld+L^`;;}xDc##VRctWSuu|0 zJQEJW>Z>x@2)I+knwt`p2v0QlFN}+oXJPG4i6jo}m*q2ux-qLmji0r3Hx-nzzn|x) zDLU7t%Rv?%&eq=)Xh)8FbYRUl6)dLs75i;Nv?x(OIQB0&#$~Y?gM(-I(=;3+@2lQ9Tf^a&YVv}i}c0{onaf^u)eAWIado^jPl^7a}PKT z@{R~4|C-XUAz$Bqa=O~E|4oYf9d@V!Oh`hS^=E9P8cC)x{+=Jsb->~p!NzF9t`y=& z0!~GOMB?n9`U~lI%!fiYLUfGyP|_G5ujWS7d8_4KeSYUqzw{d?bGThdt?^>Z;lM*S zaZ^FR3QIHT<$Lz)ZJ&z%i)HNeO^#*?YGFNbae+-9ytDMc0PWIogsIx|8f*$7xF&AUArmS!gb3lD>vI&D1afhkSNUE#W22`{);LnC zZitTER0$_o5Pr+5Kx&e4ACUFSMxQA3tF>M?Q9Mf#$h~QstMSZ1+g%zqrnd}G=Nk~g(Cu7s4w~_V+W|)b+ipkG-u834x+OG*e zFGM;QeTBnw4gpz%cXJ5^F%@O5rqg+O)~2&wkxM(CJDboB3?7u*n=e3f^T%zrsx2{; zz&dSnKFjTYZ0EBbFDXL$d0&|jK3wo#_QRwvTsxg5&fK6~tojviTlghg^nRO~>{8wZ zoRv|7*-|wRf^z`HNrhf#UxhUT<*aal`^94=IT~S%E1$``sRGnOH4O;5Y~HxxBeeAQ){vTLQP%_P*20n zL0ib~fW1VnX=|fQ`|AnuUbV)X;~W9Bt@9teCiLHwwGlYE3hIol_b~@BinZpO109t; z*pR3e-a}r=9s<&3}wZzljoc)wM3T^&HxgTxI%>~0L zlU&vztm1ClR?YN{B3hKQa!0~m4qKr-ZTroEFndW}MGS$eiqg^PuuodIcMMoBK>_wA zsPgIJJ(Ehauc`stWCJ$`hF0=m&dbov1>ZL}f5L{nu@nvE=t*>YC?UWkz3UMh@zvh6 zOK9_ObSyC_d_i{H)lDduAv;n5`$kb4Dm}^3SYwx2OQ0h2dx;N^MPKr{!QR!^kenrt z*zxG62-7(J$`)A5llTykofK;HWEKqW<%R>>c#ba7W8ohrY$72n@kk*KqK*_t$cTA+ z9=EAHTk>R@&dH{2yRMN{+xyOgUTb7>j>T#icu zABF{$_q#_{Kv%q9Oi*m4T8)BZ%Nu;vt3Ij-RQ9fNmfOEztKQ`|1uWr&CU23O+tgKG zJQ3WrkbkS+m3RA9PJ{OHWvhQzk^@-!?YA}W3iQ%5jFCNPx*^)YxS(|}7QeRkUFGg8 z+j);QzN_FLshMnDRCax^AF3w*{;;eq*T2h|4*`9~nsclo>*Lkl5T8#Wil4HU=!{U) z3L1eP|7F`4bMq8Rg$L(cy#~?K8g1n_^@eS(08!{{=zNqRZB>8LAA!qw8@I$vWN>Tk z?+WbM-4hW|x4kRw>0n>cCUwN{q4s^sF8Ap-RR;JC^%}6JGIa9IM8D8hDTAXC%rc zWPdFSU|V-S!dW_*BS2NWs7I8$KAx3|mK%Hi3GY-0u>TR8^hM~}*Kbq#VJo@7Y&zS? z?0-LFr}ASj@c|D(epo3HV$bs7iEf|@{E=Py5oFRuc)bE zH=eio95mF8cA=_N49#T-YztY2^hS?ALB;K&uOsgVUTBNiivsdV)^y2d7O{$!szWB7 zqT}cC9@^zRFL-SC@boXoNCNmoVC5^(x?Da@)pqNv~CFtZwpe$kSjHly0!7@rg;`DJVMUT15c_?T_>zu`lWHs+*v_k*Xmwy>3T*b?|sVw3w?#K;C0fSK;?W_A4pkn1APC*lWlA zdm&}^#jv&PA2eOO7eG8GvkZZ<++AK|^5eOJo{D(TCStu#q1v3b$y_}!*ON9C9Vj=j z9Xf!+ zx~-}!r|NIEY_U53bEv1sR@Vi&zw;pGSJ`44F2txb^C!t*$*b0oRd;h8*Vg&vmqYCo zlYJIezOgRQ49^~z#@0n0y^vt(f@z!T3WAVw%huNg8rs#vEpGN-)EhyPx}oYH_gy9= zDP&8XlSXGy(7eL?{CXAOa$}C%KTs|~YqW+#E2x@(ZFBT5k6c`k6E;X*`3Op$v865$ zVOL+cEw|UjeIM-YC!;mIMGmBpZ*o$~Ze$@MIt}t*d#;n52Rf^HD6}Y6oe<#N30)z* z3M#kWTY6V>LZqZRS&`%O||u zXWMNuml1Ir0j{U&;<`&>=4$8ZfWu>f!x%A?n6%mKSWH8uN8%;)@A)kK7Y%vFiLeMbyXidD1TB zl*Pi@QZCTvcuNaHxE!FCyTSs?PSlrvIeNR!6~(Uj(tkAmvlZ_>TsE)1<5)ZqnT8Zz z)!yy*z9?_?j<8^>-tE-?IOgteSlzqJ4KJ$f)$v6JmAIM5uP30aU$8arF8HXc2hrub zU2i7Piw3T&^)>!xZ8YSXvdWF`4m9&S!t$Vkl1yDBaWDWxlREYPccL#F&EckEz zyUXF~>au3NVsvY zi;2{Jn}|h#S%A}=j9z3`IptqwYel;_9b<|VRG-L{?X1_v^>KsE#CMOMtoI*w#J|Jb zqrQxBh?0hC?0nSL6)jRX;fg%dI|+slEpZ_sV?@G#M2nQhNRaQAiOnlqXT4YLMyIvrR>v}mrzOjk zSD;oKqD+RxQt|QF+FEb274o6+*1!UitrU62$5#Z^ zfZH6uWGsdqX7pJ^^`K#bOfXgXFSZ0Q{>90}jo$X7AN^DQ-&$`C+{2Z!&9|1r?I{(8 zu(@ihCdkyVzI+<-;ufFG4tTEiTMG^)v%+85Ht!a1Ea93C?{@Nl@*8pGlyzOxm*^Vob}xr?_+TgeRL0wj{`C1j0*cINdx>440&(FJ|al) zu+K<=%^gb|$^z6KPH0q1iHWHlj?r~Fq%D`?VbmwwHHZ>3Ps0~_Lcn*_KYkY1|6>7? zTNTT5<7;|~d5y-RS7Is`Qw!Nc9ly2Uc}!7izC8VMbv$`%&_bTf8a4{i!gx-qmSQNNkOL5tEe-D%tQA2Z@yA*cjim&rm zzw{5=ZNC*n`0Pp0MA>)ZH_D40XKv)S5i*%CD6cyi1@ZDi}C z%?`4p_@6t#rWtk{~MASM*GDF3z1Sw+UfT>JzA zxx?$WC1xTleZty(Ea_`(TXZFftL_V|pmjuT3`3arcJJ6JHe8kbBR}Qh*B#A+t}oc2 zH^ImL3Nb$1OkXvKEOCUsAq5gZjFDSx$U(y4WS0$Rd!976`#O~Kg?E?bsQB}d0P+d= zUsRaadTVa6(c4lE)kVOxZab1|gLH%Icr?4?VSJV4lX>8=JO$#H$9yk$_1p1;Wv}u9 zDzrhZnhP=(@E$@T3(dgoW%*$>XJ*xtLSJ)sawUtm@)ziR$F_&t8dfy_9PUDneLXyx*5%tC$KB zjmtUqTmQnZ+ zu3KlTR_8^hy%ffEtD{3+4onxM{93)b+`8G?-~YYUuZ}~r6`Nb^9M)v}a@4_04XYgi zpgR-?3QO{Ytz8{N2=bw4tTAf%Gf_A{TNh;+-cgK9)9Sd+vU};a^;sn%qCF!Q)zyAV zoXZ-Q`EKmYhSdc-NSzak=a!tsql325|JKE`RtIjjwxbUn;pWvrC_wgFTdoEBCX*C) zZj0AOkt;n{yDwxPai6#4!Vor(q;=Ne9f6?vu{}R-ao@vE|0o}%LFc+6C0_geT<%OKZj#p*R zU{NFI&{I~6zK8<|p?5NVNew?`<59n4Kb15Kl6oS_^myxe#|YOZqpUo_x)nU@Q#suS zF)EzS!ztq#iQdmTppU?r%=nNTkOAJi**t4F@FA9&%h5p9qNh>&+WD-4GLd^SA61^1 zB_@|J9FmU>?VnWovqa=c~MTbaM`M65~TMmxZ-5$>{Se-|euobJLmQ&-dH>}T=5<9Y5#1yeLwE`HpoJ}Eynh*J5tXH?( z9w<^M5fIwFJ4lQv+ve5w4&sdw?ZvIVVcWe@uY$L9`j^?-Tgu1?eH%sx_W zqhd_3+3d>UfLXtuw^5(Wwuio8N4zCyv4@Ow*`xMHqh$5o5I5E>9RG^4VL?|2jrri zu!bm;+(>Ace{Jm@1x?J>9+k?zF`Q-|1xZ|9BIk|byTT4!ql#v{E#6vwOeM_%iNme^j)JsUWZM=n65|Lus3R9MXjT?;do{61@$bBl z)7F_|?A_Dd4PDcn3#IfR*6Y963Ti6E+YtuJN1<<#PCqMB123Af3QYJLF67 zP)VOV;_KYrZig#yUM|^5tAUm4#;^iQEq$IP)s%pd@XI?T9 z9!r?^qLvKZXjbhd@FoB(l%(>j|g#ccX_*-X_{QN$o0d(mcNNhFKIUNUPy_Ptpi z(+N8tWg-Q!(T2{#ddg?V%$e$is)8JKU6#Ax`|#l}*+u{BZ?@QC)RUA=LU6Li*d;$; zU;2k8Te>42Z8=I6YuL-V85}ZJ`2dG(Ilz_4J+7V6ql85~fymuX*h-XXRZ8q{5Y*LN z2kBACrg>-e?8rT#+g9D_=z<3seX1kax;y>p7uAB})t!EgN_0)6aVOYEf5Ymdo<#O% zY)$-<LAD*qdv)uQu{yox?CjWbz49C|e z+{GRatk-;J!77g&v<(5DlLW<0pgfPST1z%k7Is;UZp;EqQn2R$ZDd>caoh>s9ucma zC4!1=_Swis1QBIht`hn+IeAW^#MqK$(rXqBNu1_8U034=#qh8+n9IJzZZAU<2fkjUNF|GNhmnAQ-3F8i=3N8%!f$W zkKg#61w*$GLun!h580i}Qzw0(HsvavinI9yP9!I7I*0A`a0zFtVV|aRZ0?|0XR-yS z`J~O>dG3xPlawq*FK8jz=5jSixpY1^4KJ3mY4iTE1Rc3hb@mFh*Me^g9WmvLcgEF6 zZ0a^Divf*9`54lI&A=5Egfbw}Z9i>G-p%09L0*^3*+^n0DoEVAoEy}&L*YfQWEHO4 zf_w+@84l-z3185m#i^ce)#~>t9A8(w00;@IWRPN z%`f&IwDsPr%x%iLOB&nkttE&gK=y|0Yj+Qq5PHz}7lDl&Z5#i`9Vhm(f$N>6$B5*2WHjw>@DJck`KKUTDv@l%5 z0fOUA&#;FX7p{1Ra&Cp)b{o`@99Kq8gLc?Q!q}d~a9N!#eS*0tG2xLYi#&ChX+vQ6 zW=rWeWXG~XahWOXo{dGBo5UbMz)a(Klobz9)?QiUYI$n59@t?gV+CY{@Vbr1q{QaR za^(aXGO^DmZKA6894dmz?BY|L-KqS6GQ_46u%f2X8s)`J^<*qmY$Q8)I$69ZUz9mV0+FJA1?30YN zxf@<;!C}UGpf0MZsME!Z%rmCeV=)DA#llTSC?*Li1 zSZ}eI(z8)psyeZ-JmG;%XL1xG)ajhf-c_F9ehAY!|Fa8?>-jttytVUp74*^mpk2s? zD8A$D`$8UqK@1fyCL9oL3I|&9#+$t-5XM7F$G*ycL0Z3#P^a!a z1@mxR@&Af1BIw_=`u7Ab5hCfrBWlfi96Jb{X#gQmhByZFy$xAa8AG(xYv1EIEbWFw z)iy?%8v%H%#B~W-yHwjv?lY`WT_rxXKCFBlu@}8iF zK1(>2ja3628aX5iInnCfZC`Jj{jb3;YfGS!LZPL{pehh|>|Xcb9BvkT}5aC2W)#1pRD1}d;8F|rd6 z`GO7k=);&sLo=@ZeL&DsX@lCAq%Qod*e-`z}dy>oV`mz%L;JtATlwt!;f_a5W&i7M|JbhN5{Mr!&P6LZ z5M+;Xb4t^A)$4y@o7sEIMS_8`RqwrCaT~l+-FwTW?;<3%t$uGvEsC6|{=LDE6k-e; z5j1gY-dpbC=eAmd_uKU0kJ?)AVz$;oC|6BV&sw@qzj_R{biYy{tEh=$>=HSQ6`q_r~d$5NQ9N z9UR^^hpQju;g;wbS007FL%x8m6mfY7|D3%5>kGCm#&O16_wjY>@JVyZ5nlS==tIt0 zXS9R}g-Gs>5sK}C&s;Z(L)!b^K&iMyA>||Zly?GGPco8h1_iAJLpvE$9&+eE%(4&r zNVBz`ZdCh+{1-8bc0;0$1jNG0XLR9Unh)m+cQXdNJsOZIqR3g4z$1i$4{>6d5=(B*{F$l;j52z$|e>-NPKUr=5*XC5Oue zoryB-s`z)sB%BTKPjTkY*j&^IlMU|189SdVGk8$ZhUa58kMO2ms0IjVyY0e& z%9^U$7tf}5-!tq;6&Tf#mMI`;kstwqk2SGLl ztqv-IW-<2M?0{fV^@z3k-!OiV1u>tZ|7myF_u%A4O4w~{0-yB^V|xZskYm6Rc^cnm zZ2OvczK99q7M7iB61P=SQ7*bX|GKM{d6xIL*MT79<^k)Ao+HpS80w@AL|Hn}N`NpF zeG@;E)9$C&3C!UfHH~ki<|OE2a*-rIF>2N9*Rt72n?kR8f=X-&N8i1pC;xvA!5hgr%_lpfiN=TkJW5aE^o z=>SyDC$^GnT_!weS94c(U_E4a=T(0e<eH=p1x+p>DZ5t|qP=J%Y(LMyo=(fv%o~N^QNndbCL) zhRA+%RMLJaDbZ<+ecF5?+a#mgQVos7lkYMfY7am`rC6q1?*zH+y50yV%5!+yI_?f! zMq&%j{`Q!&KE7_9RadU529cGc^Q?7ySBS94cnSL5heEpd?l>#`#gA7n%0cVPh3Exs zF~C4VjA~uayJ)qcYN205Ejye`g(!I+sZ3C!`gUQ2!W|j$>F2r)rYlGc@0;287Ia=jlCgPt@k;;5Tbs?R!0rJ z9A5)pKyzEY*C5zsF9gk+sDaTOmsg?(w4m?vYwa0NJid#+Vr$5mDd=p-NsJUKWi?>MY z!2eEKUqUSdhcdfK8%Pl0TX5!jU%}hp0*ouF+EV@CAO!+Dlo0$4KK L#EN)cG!RH z0qqRhXf`e34tc2=rqC6yHQ6DOZCD!4dvf~Nt=B}YG ztND;J*FiRsQ^zP6G%%j$MqT;+VS#bIMnMK-B0y0t6AX#YCSK zIP07Ez*pDXOa-Bwn;g5&<^uKh57=C8>)?UiyFn~0)AfhTX-~Z1$DsON;SXg zp19gFkQw9&sJkaAeBeiLaZivU5?pXf_NYkZ16?z1o9`*N8YKV+qAf=&-qo_H ze$BR2Ww>9)@o3LQBs2mPPPc7~N>bgt$YDIgx_3mGD>D_)Y23CuG_$o$NHp&$2uV>t zyZujuRo0u!^i~aKr0<@frJx~)3hOsejpQ*W<*hrED=qYmR5QCMrn86c39Qd_CG&Zu zsK1m#)d#mo4d&NVe1;oIU698j6*kD+qluzH^j5t z!<*aO5L-Hk2OHwV4f%01k5Eg)xd;OLztFSU#vB+84Fk72Y#bX%Ck_Ug@@#XIIg#eC zMw^3$<>bd~i;s-QHCXKc7393R6H zSUb(jekfP|u`U~l26Dwf)f&Hq{BRhJv7}jgQvx-#&PSrmPlzlip+Ty8QpR!BsdwzU zGU*<5ZpW(mNy$)N`I)(IlqL?9%S7*}$k4QUJ+*WO( z8ea+PWu!|K?d(Bm9vHt^f9Abl!~fi-tJQcFt@zofuYD_0W3a%?wLZ%GhWglM^EN$n zU_Tb^P?mGPUk?u1c|Rc~&Yq|h->N>Wh%V&FkKin73)yeEWlj!XjM?cahC1R|_Gk?g zwwUK`7Xh^Q+a>Q+!Ge~eAB9%d8Y`dm<*IK*A(iRexGj4V+zsqXOj=$)gtv;y$g$u4 z7q;g~-ruX&bt;FmH<6jx`{VMQ(6#KaD{ROXp5}T?H zCeX$e>rz|){=jZwMP6x6toi+cY#x?R$S}T^JtpX0QC0AZCrnf|yF3roVsW_4Lsvgls;u>tr@xDkt3p~txS%~ql#kS|H ziieQzYiCx+Bybl-oiWLA4$4!Y6j$F!&(`bP!8LdTYN_x2f&66=FWY?^i1k;@MUWa3hGiVj1HJ`@Xf*+GD|{j4n~yl?*#n81zau0*fW5JQOM z%HR>x19rRWtsm<}nP~UAutNb7p0ZW<`mL4~FYafcr6;YfDwFwUN!nqnqs)c*#g@_P z?=6T01pRSab8lQvq#B>x<@ri2sz z@U*Ru8b~w>_InBfy4h<$)ev^jHbi~+S*Z3FhevXF^eFL3@7I&on$U?DAl`oVg0uq1 z(|a8u=@f{pyXMfnC7X50sL9%W@?Y+Hlw3dA1;jgfRufe0*9dZ>*TJ?&w}md?#+{G} z4ACymSm(V-R6Vw52t3yvtp)3ef-FE2KGwt1PW8oiH?83*fV>^Z@dwEFXG7Ve{Dfg9 zhOhO=(v|dqcF23|SG2Q{dkemwt^K1NzPBLjFZbK%I{=mmF$ySQ89Q=sU@1Jc_u&_^ z(~jO794z7*NJ&X+j(PtGQ1{wc!b30#h3Gnd@40%nu4{FF-KQ3jjNrGe({wzJV4Cm-@)#(~_g z5E!$$y90ZAq(I4L#39^k=W{lH@>e#WBWw*p*oEAX;>}2;7Ea$nuF(JsRLJ;?QOnV% zG}Q?G3ksHC`>+ENaT70~7C`>P5jgp?5ZXpSvrkTz4iUstL!@=s7tE;1`# zCi+4;_m%s5U{Jfa>b~GscFQTU?!H73vR0E){iLnF@A`MUyec4a zf^=DZ)Yc__jgu~A?;7uq8cNO3a9=rZ%4f1RD;FVtAsehQ2QM)e*WFi6eHHw9FT#eR6B_xHD7vex*@mD4MV-Qh{w zd|$zBc%Na3;cGKnlehVnm^n8Os!z92p_Jw4*?wO+dk+mOU4Z0r+wODQM%TjW-4}6t zEf-`*&U91XV83mzU__$LtRzI}jIu%+(pJ$gx??d3s|o4co2_7|MPlOcVeh_11&6=Y4^FMC6BarCKrz>H*^!gm$#)#Mq6q+j5{Ssg zV~VGVmq_^S_@%@?SYo`RmE_+NWQvo~cddja zLKq3%ChqQ>|L7kgg1Vj0vxoE!8hYMm%gepfF5DNG5_%#Q%KIiVzg~*%N_WF{$j!GD)#PxXM8^5M_lwq~17m=THs`N|v-%2yD1& z>yduNq8`e6CD;OY8ZOcMqTjO<=cM!DL5H01g`?{^oZ{+00AfxKXtq#t@`mXyx!DGbKfLd&d3J^(9 z#`0!GC*lM3l1sh%It^>2MVzz^*LC@#L?L1|qAU)2FpEN(!r$4v$YINd31DjZ@F3{|pB9Hy+HWcfAMtqVD z=M2#rVdCjvAp(2MMiMNHZ@~Jv=Md|rYWot*%V@TRk8HLhxgw%e_uJ7A6y!_PVLO)f z`nrh!GZy_ipH`OMj(cTz3aj~?NS=hT!O3b)(uw<1FM z>P=;Lo9J%3s-R<~YR}o(1Sh0~^75F;GRPEJ$0Pi)*$)H(AlGkU^lqEWq8&XXS1JiJ z&PPj6@GHDCVe`KC2sS*u(1iqP0OyW{s(}&|7Cok2^f|$+_S#~C5{cWT^}LkpNkC`( zJ)#D_g%GsPm0yms)1W9FLFEuxjx`?oie1T*R4fU_H>_lrAgCU@nx~ZpMWC;&;e-CHOPuJguF^H@ zHCy|^SZegJ!KyWSOT45ZAlBtjo6+*@+;2@WlyYC_AtFM&t^Z&!LW4xkux9@S3q1+t zy&*;}qcP^B#Jb2HPFagj2tw?nZHzXLBiDMuTB|XL=@xKq_Sy$zPu=#xa@@ONS+;yI zsFFdlLCD#nJ%I1-29#~lk^8{A%S?wiM&)DM<8#sV#W5~K_t z49q~7?EXQS>zTS)$;33&IxOEGIs(g`7Y7Qmit4RYl=bivO)G_|81=R~zkO&MhQy%CD`bZ`>B6VafLtWtXC? zFi4;{$<&u}H!)nZ%TXaEUh=VC&S?^}5MTAG3q|$xU0um8kg?d+sF2UA^bYo+KmqNL z!K*&xC_s}IKECD`Q1_u^O%&`;DB>AgU6r9M;p8h6Mg50ji7)^rl;E1Eq7G%}p^zyJ zA1cTTRC&PG=IC3x9<#;|1;*-Dcv(^eeW+aFKW+LGYx+=JVZ}C;cl-M2cpQL3AN{K4b0vYtOFYLB1_#h1yJyREI<9%5TXwwnt-yp?}#reKbxX->|swkFwu_ znd?nZ_V?_@KB+HSiupNZ1KAIR|5G-U4S6!K((*OF;J%Ctc=9WxPbC< zg1dvj$PqtaT3hm#&LMLgy3A-9+oc z5j~f9CuutELhiGozv5uA5DOuTb48?Z{idYm7rh-1xnBB3pN6apy6sYcBKf0YWIhQ+ zwUlKn)tgLBk=8Cdcn|%xEyvg54$BRDl{0iD%An6gjDP)QFz8B@N$B`$!8uRc)$AEs zM)4Gsp3)lQ0wYWACa+JU-+Rv(u6zHA)is7}V3t+1)s22@|1ZLT@qbp|=r|-vTq27D zf~dGPjb%S}lVUu2EmS|JPtiKoMmtx^IqmH@tg*2mj3NzRmo19`N(!CFt*J3k*kkBq z)_VoXXG;vAehyo6HY(x;XPABZ>yH&Md*_)u`Go6LKnQ(+T*JPyl>dH{OX2T){$+w$xqt$=-maQ zB8#83PG1C6+Y{F93;F3y*6To#DHD9zms{7>r#Qm{6^!V&*#ZTT8p@vGW^rdMy20TX zn@61se<-%0EIl>!YIz#;5r;qqMs~O{C_=<^K}C-?7Hlm8mLm?;!}z#ifEC?8F*^nO zdn^EOO5zL*&>J?EE89oV@#EPu#_!Ufa_egeAkofnBg-CyE}52Io$=hhFMtcW_M`im zh-qk2dcH+<`Qtm8tMr+Y)OP9}^DS1sh?X$EvT3jQ7U`wm{--88o8!W|$%E)kZo-Uj z#KW6x)+-Znw_w3JuME?qI03G=IPdF+s20_yH+*!DUC7DrCdukTjG_2(Qkh9Ua?zX3 z*8aTN7JVgVYhO^_f=mA86Jqw3qGd7q0Xnhpm$U0OOfioNEU@Z+#}h7zk#)~$tGhoAi83D^DB+n`-(Rlk zRt1%``e-9D+Vjj-#)xa8>{A2nATMdlX%uS$$E;f_&`*@G0ZKa4?o;D&!#4B&z&PBf60W=>p(3UL z*L%A|CDIvzOXvNLW_3tp2~lFL6>hCN#wj-*QcW`I-Y65Ma#iR{)>oC4pgz*3*g$S7 zR}INTjFGrN{xvRC9j@jAO~uwpk#Q(jjR4I%Y~=oe4)!3$KOB{%WYV4jtc~993LUXk zy2V^F3B!2wI6<(BM-mX?MoigJU)!D%L-SaUjpGP;dwCU!uNT#&2!6qiR|6I&wjzwG z-58Eh5C;c2C0K@!^-{aAZk}ADu6S@A~{Reg+&G5Y%?8gpesv1uQ=hGFi zCy?Nr^X(&;CM8$%*n@tG8duGNR*F+T$-viX7Fdv+9Iw6#TFr@#JGh@7RQR-mRW zUelQa__g+G!CoyQMsnxcwg-Zg*@M)t7=3rdXtJLmmsl()F~O`eXU7+pv*Siql}+*n zN&Gxzz1~9s54Uyi0U;-D>%LU~_>cc+L%F(;b3(K-RP=2a&Mpg=U&8v@p`0|nU`n|( z^8XU|{;*wMS-gtxfpD)CkFfV zpTx`%$!A`U#F5ZBz7o>ql_(r=f(cG=5>H|hhoFfQQBY7&P*Av_prD|jprE*)?_O&? zkSUJzHuuhG82_GUKYOpe_u6Z(|0{sRjS}~S$0*KQXl^!8IJN-Jm= zm!73+a`JqW3)=;|lVCFZ*Pto8X?H7-vaWfS!?~A0P)$<)5ci{{-!@Q+qILP4MSJkL zrTA=nn1lcw^4*3kqW?ARZOU{5O)kh078AznUH&T56F6b7; zRw?Az_`AI8#kF%~7P3RUl{Hl#>mK&V!RDNqJN`?>&=SpDijx}VD{GBs+9bIXD8XCX z?+Q^RuF%?jQOOoH2y5aq=Eq^%>ur=pt}`DKG`;V4{>ufC6*srk256aa;htS?DcFddZhp!%wrha1RPW&$Dvyu?vGJDFGreyB5)79R{ z8-=<6?1L02|88fhxqmOC1q6J`CgYhr0i`mO9&_q?OprZQI*-pWpj+$sN^PwjR*!?szH zL+H3ICDygDnRjAm(8?+7Jyu1@b;2;iG;xKod(jc%tWrJ|sp)=pbck#6LGHS|8(Cpk zjfeSuKYE=<-T|4cRT|QzHLD4ZU_aF^Y*~#HA+f0EVlS8HGc+<1)>Oi6MNK&cFVxz~ znsSunV!f)S&?m1Bv)Wti*=}p%duUX#38@yy+M48;;8tpaw7?}|?S=fEg?1?!zV_@# ztIh9`(NQoJa#L61l)}%J3%|v3Vf8*A9Kb~-8)_p_mLI};;Fae){fTglIVvwbzyEygj%^wpiaf#@usdc{!pCj7*{eqjB+4U3aIaDp1F zz{0_dn3DKfnpA}$Ou;;kWs{Fe%DrPbF_e4CkUH)giMi{zPk*Sd8y7Ag1pfB|S^9i@ zwAotoiR_?Y83jiY9^xDAbO3NZh`Z|Xok;+&o&f)(x5o>T6F24UqokwD@6*wT)E6Qy zSbc1!TJN)kETCQzs>-C2C{Vmcdk!77xmaP$v`E$S)t96}5J4%~30BkbcY-6K!!0n<~d&?hDd26A2F+ajlxE+{IjizwU=+sOv9wBVPMXgv{fwNBa-j zt$w5I^)#>`D3xWOr zXNT;XrwiHH)Atvum_410kzYq#0d)xyfw{Qh)jI_2tu*BgPygy!o*$P4hU7N-T5KYH z_|wTDDT%oRVB%TuDjSCzQY2?{y!R01C!+nq8achRJY5KBKjx3trwbwN$EZtUo`6D3 z#W&XO;~}wi_;Xx(B=YTz^>K#4ZI+!rTcv}recq+2ksY*de?c%xi}h9uD;CHmtT zxQXq zlvtjv2K<=!b?+l8l#G742^&W+)>bnT;i#Y^y7extf(*K^%TCa0K%8`1Pr812vP zrXvlM+HPg%qer^zb}k_@$v3tXGa~gYH>}hLnTF?%x8tsmAIsgGmGg|u>Rv8BwsL6R zeTP{UVC+FGowQwYdSOxRp~FBB87TQEAs_A;lM}x^Qw|>}*lWw4iPL_F;(xj{mOoRm zAzY>_o=HtXPf0zn@)_qW4ya_>s_cgZrmLSxl2b)rvx{Pu);!}Rr|_e?C&7lb*)2y{ ziEC?~iCyd&<7!oA!`k@DE!Hju)g@?c^6jjD#*g_EoF^zu?FYtPfU{}v9^kQvus#AK z4LNSCpu|IjrTVJq^Axh!l1SuMt3r^OU75Z-2V6or$6rA!E8}ngiu=2(a1AF_q#qK7o?3P(A~`TnE$|A zk%^7R$Ayb#26B!(?hOX0Y4VYs$aSD%C|248u&Y}|opp95s^Lo!E>hloVW$E>MMKWm z=^ULf96J+Bp_!2H00P3jUG!^{)vR!sw5jNpN)KG6USfPYp1F68^j+WDj3a6(3OMUM zj1qRn)aG*AC7umI)#l@KQ5z|v6qT{yupJuE`MBzR!Fu)EVvY~(9iEg?{WH$|WRZcI`&Y1+5*V;bsoZ9!j8-RDR0`CYXZq61Q`(BJ8`GY$<>_ zLOIDhiLjBN>#lDv!UYAmkJ>$NN}<{=yYC|(RQl-=dyq?ikPwN72^8|;tUbzCstjPy zmU|Ivmo0m?kO6Gu%wzepalj$Xilb8q-il`nSs^i$=kBqU&jwjR3GsogicdTUTA3H; zZS}K(j>=<8cuD?bp(#AKHolN>Dklgj32UAWs*x41tTsOa>6lraf3!G>+mzJ62f*xl z2fetUT0`{eHVsFy#(c&Ymo*ZiU{f{vWf0Q+3Y&d7KCbJu7M~m^?*|sG)n8E3)7lb3 z^mSgDy4{CY+HbTR32yJmh;sZUXzxImP1{U7(;cEDD&kaT9nzv= zwYda)_fDKTs@4kK7lkJ8PCP5pFqk2w`PJ%~+rg2FZ84sCs$Y$SV3OP4jxu-HuIGaF zjzThT_^+RFK7^{=^!K`M?NSzx+CPRr2I&29C6&^Ua@7h zguqA@l5wDmS3*uZ)|mKkOrtO*TWj`4|c`Y z)~2w6NlM|}HMQjuZrN?Mk2^x4?UzHHf4OB(ht<~>61F&bkf7IwYzu8w9E@_uG}Z=n zK@KAC1QfU_TR|D<5iWyF-?il|4JyH0i}(K0EA`fz5Rc%}`GK|hV9K_7)Y|hMYDc8( zj_8$hguv_`&U<~CIImZ5dF+f9MJA9Cga_-seD;qLkk7gP&bn*;#4APZWr)%DaF-t5 z(Ce%6`yJLFZ3<^0RsoVNerdr59s0rEK^w_kI@pKD+E@&wutpR$fP~XVZG|V;uZmmh zcrWv z&RvFE$cGZiobegsFNsakyau7LQc0u=;>laAO@d_$ke2Z&Z| z%$@2Pd5S?g^?f%`Stgo$*@^Nsii5kK4^(|uEgnP<$}ovu91iGvdzc6aKj!g38M5~% zkV|=h*9DnC?>TDA)+OgJM{gJkTfXked&{|duU4~ST^x3-a32V$mFvpkBV7k9#X6@Y z4v(q@5TdrO5Q9Jc(bmLUUG&Osp;XUn9gsp}zO|a)42wTWA@Ne`)mGmsPgGHHT3tMI z8>{rdGge=dBO2|#?!bhOHXBeM84?S(pZwi@_IPi_N zRO2Z_jl|)UkHqFUDh5=iz^G0lWMKQn{Ify-uC*Mtb58j{i6 zJ#fI?@x5CNit^ifeW?zzMVy8HSSo3SMX_8`Avv}UR&S&s7Ipa|DB4KXE7|j*Ys#TN zRz3TvI7>tc69GQLpF%gmiReKZ=d+CT1zLlXKCRKB0rkIs z{u}busl+>&aMYjcg8Xx1+L@}G!UVyN{%Diyf&?kk0F$2bb=%Ww(+LF~DUF}aWK-*M6m4_bfN9!THv^<`7>8elRj*2e~7Pb|(~>qAT_5tq}& z?{QT;<35K+aB97JeQX(?l3Z+B)ivuw@)@+X>w^{H)RYZc=nA6Oh!j(I*!M)lB0?RP!#HkSaOWiFdiq@2V+G0&zv9y z_D=ZRi86wJJCzFoKW?WJeTV~>#_f!cBKbr`Sc+HPWI|K4%c?xFU{enI<{dU2Z$$=) zH*|rc`r^kTs3|?n?D}6_9NS6-#C$Jb0Oq2||M_@UZs{`T&ut;Mslq1QkC$XLoX9n_KO!|BW^R=Hf9jz`+<-}em`pWC(h@qwc%*;bPu!$A_`{#?p7^57Az zq_#jjllns305w{zKEK-e`?fY~h!!Qsq9P*rsxfCknG?BXHsu$a6@k~BK#-4x8MI^v z-22nqvkF{n^$);`d8CQ9=gQe?@38jg3b~ZK@KNi?ez*|XmA&4oZKHKoJ;FE^pueedXo$=q8}yM$ScRL_NH$iABe_M|SO82)BkWVh z0sxUasmsaJY2)4jR?jKSq0t;x7S{Ge_I(6v)ycpM7j!(=iD&+vc&^a&&AWEmspo3aoP3$?e;ws%LWl_8^4m=klg14pT0m&qTwt zH!JhwtS{=~d!X!Wp6&ilwLv9WabY5~Swy)f#*h_#(C+0t(d!Y7bw9RT{Kl8| zAomMOgex4y*aMo(myLI)JxaXTgI9pn`RNpQ4Jg0JsI;suF8-c_NQlL9vOJnardr8) zwxX__8ocTyfz$>4kdh1AslQY9i7{JUmy}0oM7wQGHD<9>mt|X97pOO^0lB7YvJqYd z&~ZE)uUKt7ljQ#i2LOGW)x|UYLL6TnSwokfOz0eZw(0qy_dL?1eok%-F?#wpt{m$RV`@;5C{dlEiBY{c$#AdQRo`|@a*A+^L^PbH` z%Hn3#S7TkuQ+7)bK&e z3UOH-y~#uNYg<$G=68@H*w^M)#hF{Sl^?C?;x3)H2?q$p zI6Apg6+lXSUz}0xbUfqabM{J!cRSL46UJwt3VvEXtx@DsGH>`hvX~T)`;Hfqk&t}t?1b;#-dqd> zOY3ytFcMRS$MyxNM8ZPByAxD*8DNaw7YeOk_m1^@Cvupi!fK2qla8}(8J;qdYt5-e zZ-wtgE@pAYbT6u7)s{~|5yx{2P^v-lSm1MZ{E26>iHp*o@MSqb0M*Id4NjesOUXNP z!gqj>Rj7pnd!!WjJROZCsLHlA$L(?^!8|gGe2Z_{q&HfejQC1<`cLIjbtwJXbgaFu z0Gsh29rAIu+3ZYtLl5KDKNriBd0JJ{N7}3JY`$8!Qf(M+_H&R)_tb*#X3^A5J!My` z1yEgZQGdfBz2Q4}w99U0mq4Nz=F7ly>xH-- z_e<%FCq`mGDa<&4FY}Ur-%i$W|Led(yBqr_VgK)T&->{6OQ??SSBOa17~?!9A`cS2 zl9N*AMek6&ATjI5hb&|gFVXh>+}_`^{J=dJO@LX&p)pcVDk zrOuH2V}y&%RX5ITtAJ9Af3o4Fm#y{1Lj1(ULM0sXzJ81W?GE{WA#a58By~hLg<$~m zB%V*EJ%K53%N@B`${1ryBeE1giy2? zpUZ5m;HVi~f3Lk*DD_C6UC*Jwd=hk3yQ3=f7*6sxP6g<^?N;o*dxhz;%Y=ZptA4%$ zyrmdZ29ohYrK!N|FmX5G4*p_XYVs*4rGg>;_k4-C2&CA^{!nxMXb&nlignN)=7bL@ z%_x9A@?GkqqL|ee?Ea{?4+ef&eaO;eV!mL@>*LsND=x#tz1EjwyGP91%6i9ax7&Nv zhwt60`d~ApZTlTFYiGE_vE&95AjB989d%;edNR zY<1bX=!i-9G%P<_cYRRft(3EZN6FPgu}wg2zC1FZ#T<;@p7gXFsg_i(=a4#R$XI^% zXIvpi>{#{*egEFZV=vK2*ztUWu5Hjxl=6?OJ^&Fb#|C9f*2f1aWgw_rAFekVJ zTz(XTx=pL7@?2ix-^<+EwxJNH?Q(2gwxN)rvP@nTbNL1*Ll^#W4#Fb4!1uNyo=L{c z409{<8TZ@eqIsORRT~PK#25JAyKMD_I3z=OeGl0he@CGZNa)%PK^NtM##YPM$W7ig z8wvt%c&X89{R@>J=&`ySuEe9eF0wv)Nj*C(2~(7xVZ(+X(Z!#u(fj=Ptd$PMU$%7}-rwD2tKuoWhSr&j@Q;!%_z6wOzG&_7W2!fg1Da1_{rdIR>l-MHwf>mt1>pP22CEO@ z8901Duz4fVLMqEEHkPn+N1`#YWARpjH#0WAAq9PjRWk5i;@`)&&hZU}q;AGLB>NHH zPOgiS-jB)xfPxdbB9LR;+L-P)cFHF(fST;IPXG%O%#L@I&SN&26T)wbItM=BcHGST zn|>;9+e|L*2S?=TGwZJwC*Q-!>qxVw?-0J&^Zr6r4s9W(rmI*t1nh=05srR#H9*$& zjE05ddEOR%9OUK^4w_xt@azAsEABTwc28b+c0D?gF8XWk@Sl*YZalFTguF8_iHvgs z2ZoL!^xTRj&^&2<$%8}5*lH`}MhF9d$}9A0rAFbBckz_VYj(%yeLyQ>cQ?fKO!C}A zM=(`k7mguZ#jbiL-`IU$9%W~@2mZd)gv=27hw)AElmwe*VS69N_MB&Kqz!thoHH1% z0nX1d{{@(HaQe$%Dr5$h_y4gKFU5K1s{4}nj}thp^j5INWI}(*NsrTZSz5lb)%i>j z*L;bOxUGq2u0hp?E!f(Z5_91>fz|7B=sfD?o417^hH2YM+~-)qe;6=K%TtCp97O+?uL zP-f581c=l9rGVu4wPhnR1(ZhZFa28kpJ1x2!>4fcFg)wt3Q#bj^QFXy9#6#8ZTk}T zgI#!@Sa-mKvo%Lw!s12i%`RFHjQXPi{s%~%xD~S#gFZpkkRFNGTztPM&ix{n@K`+4 z>3q8g9ZVPb>qerm_7UUdr857~xZDn#fIr6Kc+qJr==n-jT) z2Nbf*eSXTZl1ijq1YMmD90g1fpX>l$Kpy%fsz2NBQR%L!><$#CI7GKi`)jy++3;o@ zmXDve*_>Co4@xTiq=LQWz?85JmZ!Mi-FB=Y$?G9o z%2A-|b|-&yK(z11G<2&ddTN1FbuSTq7+w^e(EaQk>WUTcH7^Gt z874O~?kV1@%6me2>*51>P?AXpsy1!)-ePf*ne}2#Q zMUBV>6_#xx;rjzVk*8jcLnd{0QN^FGrlfmcmjz_u@iTPo@_72hCIhJ)Irz6^r;)BhzU7OPo4f3qFXXyE+Vvc42lCsE_)M}7?$$^m7wl#ZhEDQxyOo=? z=df;t+x~KKa+A!8OR)w@^bIScXoYesPTQT|td84;6mBqWcVkE_8^zqH6zlgKx@Z4n z_p|>}OyxoDj_#;(+8$P4DLbOM}8B51yLF9;qwT3`N@ zIxvqxdPV$F_VLTMvcb>2Tn@gXEBQ*V%C9#MbW=^C`g(yWt!XGYt%`1I8_JQR^7db? zCb~g5DbY1r+}eal$(ri2PYwnwMl_e3&3E8Gq{JE;oTj-T4EF;6QV`A>qxE^OH1Q8~ zgVt0HNVH5jjbB@H^dLn3%)iyYAazT&miW!y%m1L&!6#F|khN7KklU#ogmc#J?+0aa z>ByL2U<&Ai0`oCj(} zrS&%iX+^RC9|s%a5*R*=KWndzL}x4qOo>}y$8rEp6wD^(oKNjoJkxMW8m{b=<4-(O z-AxHl$Fu*DUQiGh6W{7SM7Wa;&SA=^C@M`cXFmGcv!f9PJP~s!ax8O2e^*WzR_@rHS zHfJd7JYz{`KbL(E<5I}B&Sx9Quvqgc$nZin3hr#8IG6{n<`Llf|Hc-5N`vI(=&@_r zyku+f;vd(exvIfq1IiYv8bOD054~9p467)7<0Rg4e1n8-wA(%evIS5sRULSU&QiaH% zDRr;C60`!c^C`r^*_K;Dz;8`$~{N ze0QxbfK{Y+@#Wyd`fMOwV#6zOoVN9#*+TQ1@DCH*M8aQd${wUt16+9*HOI5_h*BsP z@l>Lqy#vrqM)mBHjD;!+(3U+&YUM7)h0)p*hJ&yS9nnA@on&w*^Ioc*Rq(8yiisclaz(M~&HSAy@x=O0nksMti zsjtbkf*h?D#8D+Yzm^vfihc`-$8}LTd1+ zjV*?(U=W;mVlMLexJW9$Q#qH7WZF3G*nyUouzr6bq=SCElE#IBkB;;YK*y${uM)xl zjZpQ$>Fi5v?MyY8d>%^>jLil(aK|t}a|vp-2bVrt-+ z{3PqK#qcfCxWw&oI(CGVz z`JZSd@$L-kwf~f^d@1++Fji*p;7+Wa>agC;d2-)Ud^&;S&O%2LC-4*4{09MwBnnp^ z;c&+x_}U)&HbhUdv*P*K^eNbOQ1|c1K7}y-Iq{aW*T3!uq z0qj@GHVAsFH&A+gsBha7(3Rp7pl5ru;H+TH=OuO)|FUJbok_@%V0&ybCpFZw-==cVQsbJsCN^Dx`Mu5f-vkPMXtVxz$;&b4 zT^t-@cWgd4UOFh|x$x?*{~IbO2s4Q`ouSbc1#@>bM;;#Ownay~_uK4ReBenoA(r?P z`+>dgbJ$2M#tk2SP=4Mw1H}KJ&s#aSm+CjXZntyE!2yiF9bdZ+BpC6qFC(O9=W8VZc_C>-Kh=IT<; z+=l8aglTJhEf!b0Tt$vG`J@k1=`lWbpAFy3mRN`|XmEGW2`!+Xt*-^*%Ym_kKDOoH zLmPkI(*GIQ1iCezgSdVUZOrrMx@TyoWgUhX8159QHeuUJ?#hE z=ij%=2-zJ|L!-ykdVOn%j=?0~|60%@ii-6r={#wKk;OKipOi2~ISw56mD_`LHa=z-;UJ|lnzoZwhj=YB1DQD{yhAQB&cmr_AVZR~ zgHv|;wN!q=JV5Kt#PaI~mGG$4KfW=OJ~&Q7Q~3?mf|T}hI(Jz9W}?9}3Awa)T%3V{yi{~orOIIu7 zjZVvKp(5z?n8 zG#0|RW0%!>TcUHw2!@G#7*B}0MnAFAMTnA#vy?$)fdwX5!RnvrU)nnHkPZLzOvI}( zo;g~%>u*zIp?-#`D~n80xPv#|;m1ZiMa;MlQ1j8EBC;%0J$ z(*Ag~SKcheh^XSniN;@*SMo{o<^TF`I~gM=YtFPy_%CduDqwLcMsO~n82IdTLiCTX z(I1=%WGLz8fI;Cy{-#1IhG8v|=X7H^V^RN}Rj0A63hP?sajw%&*HIG7Mf_KIHqNCdf5W#D?Ie#9- zTdLcxS0fTc(<2t*Mz!2CC0@8B8H#7;L8eN8sUZ7~b9p<@a=A;DAwt5VZq%j5U@bP` zj&sL<>@PJP?k4D)qzt>4p|d@v;br8>*dt#SFWMuuP0FwW~_KUu-}9HA!jS|$>6|8gv=9{y6W|E zeBZ#EVYPpZo%nx3Dc8L2)JaiNI<9D5Y^@K4PdJ(m|8T>r^;Vm`4#~k}h;iy(kA>PO z{c8403KLx7vtM+J=?TrSdNFDP;Ibxj&0gT%m5Rh-n_BjN^ zOv=%_`*pt;+z~(--bgK4gyG)!*7c53%}^62o{4^Q?9o*crRP9w7oE2eUs%*_yKF3% zi=vvipXF5H9F(Nd#{D%}L;Csfcs4#HzWjv$VuXvRf=&jEg`NYQYEC&>oyc~WNWZsJ zIoonOC{y8Ab~*vt@^5w~@eNi?I*okQbe zy}e8_LpGP~OL^8}1DcQb#lX7|*rrSUYQPAWR!$VY+qPKE;uH4lTD8VX8{+!F>yD#G zaKyinla!3l@e>-|%x5SVy*XNVYV}&5FoZm3?QZ7)3l5pA` zhijNfRPa~w3FnJ+$!|EJ(J=G2CI<~`pOUk#^%kTalg+#)f9>qSPpI~dAP@UV#!>f19A+$s zMZ-{E_4XxJAYDpR*6@a(H=UqMJd^s*d6&lgyzdBsp5#Yq%Fh3lJE!@LLL7_js3l;= z`&an_ViYag#lrN?T3a=%DP+C&HEybTH+C0;9x0bH4XDk)uqh##BL4Pj9vG#$!Q2{6^K&ITm3t zJ}6=##8ZhBujO|Bfir*ug9m#{5KfNrbKHp*;@TA$$goveAY5UibK zOn$SVQP*If)nzwj>7@Q3&qzEgR_M5p8?v2PF0e@{yc!)MX+aU=t*L4vR}x|TtTpG< z;C6ZjmBOecS}Bp^94A8Blh%Nn0!Scfo5Q(-tRvo5dXpm*ZYyf$j_P}*HzQF1R{p)d znq3FQ1$0IO-7UncXkNB2o)rv)PsgHd-To0=FKg3V^(yO~j?9$xzgegrmG!Wzcw@4! zLxxA*bTV2z0zf7&a;N-4W4Y}Z>0h>ExgCY=9e)C@?;#Q}F?QTH0wunk$n6~*ke%sd zv@WT5nC2Dq&J!_Ba0C%08hFf#R(sk`=Z-@s<)nNj+Q_Zv`!cIpn~Z1YNz^_=$CARQ z@|hy13V-~QO*`=G*Z=9SAJ^GTw1{+K(B&&Oo67+1DTGrMnuLclHkT8}j7_adpQQ4q zFT@Lp=`(gUr;JsYQ?i)QZ!Yzg?An{bFo+`rQl)nG6#NKLa4y%YJ=wn9Zuk$3Q5@x) zzH~cx7?ELov9w;OMw(Jg`J}jL zC^^E~Y=-^^A}dd@nrK$24acYi@-_wIA>S|Af$DrVI4Jg0yrjPRMCuY3Y8!Giav^>L zxxU0|HRh~_|7}z{ho&4oa%h3^%|3^p{~IcUmYB3d;tE60huYdysNCjl))vbuV)eDP z$Dle;7a^$%5$k9Qx=d6N>Cq7b_eQ_+5tcjWWAj{R^ro?dpY|6Uhlb!gW)c9o)H2u=Zj{-aqiUkYjWQQY?G6Mgrc!JV z=vL-^enSI?Y$2EGL!~RZ+VrcJ^MJ)YQYwxJ;TC;#(2v!>mgv~u2V0Cw>-9vWqR7M| zj@wwaVe%y@!Oa|nr1TWJjs6GOd~9A>yt?2bPJ*f z$vNV(t*+WlF+MkjBKzKo^Pm_K0%EcWI3R|Y%so~ULo44ZiHhDT1PNXUit$#E5Q^+6 zXv^x~auVp$zmh|G^_x_9HslBJ1rX2?YmD(^7@cLjA5ic#B}gS{gEjk4jDQ4Q5#N_n zKiJwZ#b^p4nKG?%R$gH(yTx|VCGkVd;66PNn&7UFf9z&HE1Z$|qglJ zQ^|4k?JN8xjz**0NGtPmWrv1O`4*C}>h0K=&Alj=DL!(C3=T z%2&Q!=1pxI^fWcmKn`~DO{bcQ)xQ1f3;!uSzpMaOmp#Ems;sUY#Ore?DtlT(%vH4J zk~O~Vi`-^S{#=KTU1@$hjz1CAN|)V|UA&0mxApDd!;1Gx@evA2-#q=`T;v-|(0A?cf|We>b`Yo%xT$c{@%#u; z4)#MXly>~>a?Zq=pUBtfNb({)8P7`EBxWp=)kGrCcCsUo{G03PR6fEfiQ*RO?eyF3 zhW1YpY9@uet^vb1m#^+51=3+{m ztv-{e>o(Up-;$C7mf|o~i|N)XkOD>4<}=i$3hCirtBGfXP@~EdjU(Ut zI|Z|ckl*Xx@pbWVuX1BBJy~bOA$?}`KCj{)Vh!&U0#o~pHF~e`j*8x=cZbEDEp;cb3^_^h84wclrZN5N@lY3gM{T(MtQYBJj->dVBf@Im?~ZIw2L>zIQ^yS_MPVq&xa|j?`^U#hPQi36$>SL)Py= z^-+?{24ks;gAm~S1mW?&olK47>MP=c(?wk>8_W5l{>O)wY|jqt&}dlq$E!sncd_OO zi+#N6=n`zTdpw_TK(MUf10V0TPeR785U(eye!edyWo7L=<=?<@;koAf_z|u)J$5D+ zkz*>YE7HTHFB%Mc&;AjciZ5{zwCQR&@iDcTgbhN8ItV5OjY>RIY%_QRJX1Co&y=I^ z3dkUB@qB&@ii@+Kg}q+L^+vbL)UNu#KmR*NaxvGg7a2&oMXvcAa4|rrxb8o;AC~jl zjdy~elUCU?!rn}zI7~eskn>h{hL(ncY)9>O4y-gky|$Ed;IJ-m;O5{y@5D2Rli-D+ z{><*?v)FzjjO|{IHQa;J_Vx*;njMC--SJh=_&5_VXiowJv!NHZKkz<_@$pS zYYy&4N;giyMUMTl=1{tz2RmoWn*;TcUsU#LMRTD-m=SP(S=n4Dj$A=)RdYG<8(wE+U9b~@#Cmzj%8I`IFah4R?GqSW3YjduWK%67%gXU)2eUw za|`VdKc+ho{8iEJhUTO*iokkOD^|{+HG0QK2C&y*&9)|==i=l>2|CT$ue?*}QPxcV zt|i;;SL|JD0vuWPNDA*WQYmaVRM|74h9pMGNwy(c*NpC{jRC3eDJ=Sj42A=aDpu3DG8f58adna!fS`t#%g#X_=;T z2B>=BL}wh;HgC1rd_lSsnEkn&2(}lg%YWh?mG)I7FqGwOKEQ;KKxG#ypudN1TumH> z-}(?Y-^FSY5FC2Z!e48SdrA3WMM17t9e$1saU(w|nzZ6^X*XluQo72Hf1zk6l(+Mi z?~#fSpm|9QAhE42Ia(D9+#R1QN#O6sSdzgAj`8PP`|ZAeN(o?ujt9{QtpL2nL+=QO zuYa>gfslx?w$Z5p`OkMKcR#oDks29QaS;1?g@1;mE6*?3XhKW>5uIW-##;n>|~wHt$QkOhi_ zax6=oe>yz+q18v9()sF`??a~ z1wNId+AM0qeH(wRDbCp?S(zk`AJu{MwQQbRJDK!c{)K8v_})%8JGy{n2Qe z4U;Ysv$G8b>s5L)bWkw zlKA;=9HSGyQgC2sQ&0N7>>xAw#BX4hYV@p~dZN=G?R2zo({IUlXlLS?`x+J%mrYhZ zDJi+jEmIr)W?&bw;%NCrvlh_hi9j#dbWB{oq~!Xsc;!ginyE&>4UkoqEqOLtYXfxy z$+GQ~cdi;6a|8=0vOgcQ&`G>x3qEB@#aE*-`d!f5udqc&?`~3%9^~-ds0G6 z8GoZ1NW8nJ^}FfIj8HRRw{j;PEl^r0eoIA7Zs*>RpNFgmOFrPvEq2Et>!E_hUGItf zydT=V+-dG-yie?Yf&y1p3Y$GhP$WmQ8ec;mAI3AC8ftnW>YlSlKGIPr%BCa&=rte$ z^66d{&!m}?N0ee#mv3^KQpiL>QKgoUo9~Kft^Oq%DoAkUrsP`wh9P{_rb4Hls(Z>- zZ~D#V7YAA=P-1srOg0s4C5wTsJX^b|U@LfyRVSfB69itMhnct9guc{ml7@ynjNXOH z;Ps=nsE;O6EQ<`Pv|2+pQJq-g?>40hs4DK+R80y$A*Neo8wUoy74z_>Pyp>8CF(zB zOEw@l+_v~7U%F4NJvst5_&nj~H`d`D!F`>zy_V=&F zeG@0-2@jw7_{J%@WBo!bg!A;S`9T?gpnKH^9z0}=K5!QW1fUeriO80Ggh@vk z$rD>r_}=hOHd8K$%xHc(Ho*Md^5>)g7~OBTbEyvyK!E_Xl)qK1oISqd-wl8WcYTXT z<+yOqAJ^1$+Wi2gvJygFAH?D+;`XdPj9;X@;#fS2*~SX}@?Q%n)ET$UDVeAs_xWG` zh6K87bHUt!!K1E=UcT8iBGeY5%C^5;oRr4=06LH80IzAnaODs?XRG3w2t8qgVqtCd zW+xoVD)X(a@hs>5PwxQ#)x@e&}iDJ#l1AcQM zGNi~ZHMSZacMJ(s!|$!pM_!zKm*s8pA0^+CHT%a3IwU!twIslWBWsOcpAqu#jkRt5 zwNXmoCu&3fubsVr`_|e6FnOH*1QyF%wPSPe4pfyY%Ei6jNP@g9IfD{NJ0H&o0`9JJ z`{J4Ar}Qim(7SUko0WXidZP)ruTum4x@`S1BT0Ba*`Tk)#*XdOlo-i1IzYnNF>gou zijkfkJ5~X}ZNsV~fZ2Ej!1u+tiqh4N$1|b8Sw@t0;Y0-mG+z6R=J;ej*{(y||?A$DqZl z3B*-I(t`w~%-`|vVIuRNai5h#*`sQva*9$towH^wDf65sGte^swK&-->;CeVxP}P^ zu@x<4BAc<3t!#06g=$j%kzC)9o0h^$P#Y4DR{y(en55!syhjd`Cbc%cWTgr}sfd}H zc;@;IX9jA0uyrriT3t&VNrJULloVL+?GfG-XDjCwV*VY{fwFRqEy0r?>KjtHfAmlM zG?KeGg)7$V{Y#W;i3N}z7$>xBjMnjMZ752jHgCLZH@;}m_#B3|1?z~$@+Sr93TfNh z5<&5mg47eqcX}sB$zZTAK6AZEvD{gA^-R`E?caIpZ3)KdAPHHyrI=@bj884$U;bu; z@#|BJ2zagLLaA7-7JJNJlm1TZ@39!6Bx_>RHwq94*7 zb)o`L(Ocyfx05lxECm;B!uLQW3du=Sb;O~eIc23em8x%Wx`JvHR5=sv@@UQZ5v1K@ z1+UVqpk)9(r{b9h7&xj-JF*p5>KX4FR$+*GO0!jKbR5iB{L)-JE3g2OTyf`f6_A6m z%;9h1dquplUCnox+KT~d(Pw$2sDZD=0$|_b#=RcDM35LOa7+ufXL@ltU0(Y@G? zE{Z1FeIK8KKYQ#!G?tHnJnFOr4;`j0|2Arm{7)Pr&9?YC7hCb~1-#3)Y>U$iad1D` z^6J+-x95tg{ohLle*U$UTM7*!;WXFzs%TK4W>)x2+UhNFOORZ|&E9LgQ&L|lYvEe& zfV>+Ak6O)^$M`9IIrl^G*|qWJJhwTROU8{KK^kFhoezp}CCYTcFx6L(3eR=)5nHwd zp}{{_?oigaC2SoKlz;O4yn=Gw-hSM%s#h{s-G{p<^9C%_0}KBgO~&&IScL`)&Z62_qNaBD>bV< z;;}pXf$5vpE^_dDeNl^L)cSJ*%5I0Id$3wcrG$9G|J+7=c%JFbA9G+;T-lDrn`wM+ zzO(UMq^+6E@A#H-dkkPZIT7$k?<~z98c#bJ&-_?AFEFvCknynI+No%wW%~)V6@z-( zM+NRt>QbF?corvjZ?VZOK|4{O783?}o^zTbA3;j-e6Kp36Ly1)KiNz?lgEe1@C@wE zY&^qMTb$43TruYZ7vCMy4MhvN46(29Gq{yhy`!fVD*_f=^%=po69I{PN&YNpTkM)I z8upV>%F{|acKtClV3JnlQ@atH<2R_DNe3xBznQIBQMuPCVZ*K1mQOKPp0nH0LbxQK zpzqnLr7cAO{Oe!sPKBc~#eK%xasfhk?#6endvYDbz35r_{C}|f-pl6g_P{rqq~rF` z|Ln&%@KLlr|BrL*wrwpm1e1VlWy`k4`RN-T9_9Qj-6p`U^RV(A! z1x~zJ&ccOc&-=UfAGOu_3*G-qwq|RgL<2Hr*joRwI7x*btJzvekUKMRdGuOs_Fam1 z1s&O1R*H0#_1W*AM-LN)7j4Um1Fy2VHg0uvhtxEAV=xx|qcwXcn+L2VA!y^8Y28|I zeYe;d#oWa=D90fL?G7HuCUYZ-SL@hX$OJKN9k$np>V-CSdgt2)Y@c`DRpjCB_;H9_ z0oI$V&5cjKKi?>qapB3AH z@@PEffJYL_^y5_vS#v$N-wB6YnR-vgYegt|zUx_=aIj01pq;83C<(C)py%y$PG$H5 zxO8|4|A0~lxExRB9+47Zz^1CtFUctY67q>nZ_T^JvusLV>N`DC0ZU~QTEk!2tOMJP zGv8di_8Zu_kokCq(--i$V6l+!9qA%<$47QGLHc_67fpGc9%U!+ z%8L5BfZL00^OKJES)n*8=DjSQ$(}7~Oio(M{TXyqH7te;C%j@?In6j%g7_=9#nr6} zx1;z2uG$s{q;{RH-d4`y?w&zgv#k&wj)ia`gJp9&byju#3aS_%gd}7JC zuch0q%LW}JAgbTyC?rhCeAdOdXARqat@}@Us9B?Tt>oh$Skty(K8RQt#THg9mdy@7 z%knL^(c3h;2dr(_0&;hEx%GIqsr-o`ORXZK|OsBh?@$+a5=^4ZgzY*3r}9}zbLTsJIw666;c`m)62~+7&QDg0KnRgo zK+!xHj9HtC311+0*57S9ekt4AjLqcq51_duz&8giYJA=I=WVWfMi$rt9BG%UXBSX< zczU>nc*bocJqkSjtX)l@ibrs>!@+Cf|1>cmTvh951$|BCp~C3~j|MXhZsqqPSKdZO`Ft#MX+ zj^GpzE4yAYHxgzqGcs;O_6YSWRm=GZC3&wf^G0Emqf>9PkiP z6UB42-rFiKpdyYMT4Os&BINb38e0odRsJv}ooqc)gpKBEZl2l3&5M$$TPgq*`z@Yj z!CG5`0KeK`ZSiG^UWFvMr?ONA?X9VNItMr35$#Zbizby)B?=A1j8mGdK6RMrW_HK{<|c+5v@bJCbp_L&N!W!2-8ShMvvspg zIpU$|@GMc&F7X1BD zdHi2ZgvX?)Ac)2M_V+vOTC4@D1v?OzN>MjmPh8!^iM^3C9HFF+-Skr`=e}Dpw)98S zjI994+pXnlE$Q7kU>rK6kR=}!FJt`fs|iu_j%Aib>TWy}gD3G*{L8(Zn7q~tf&bR- z`*w6wXp_tgsO%o(>^6J-golCipMl?>*dw1sDZyj5y-=57rm$>#s#(5;+gcvKC{Tey ze8u)Sx5PbvG%y6kTMEdvX!@jS#6(Sf@47n_S7KZ0&Xzi*N%u zU^O}HFaM78Si3#$V+^-cAg$Z(6wj@1T%w%w`grEzs0RZ!L?;yLW#JmD7EU+Fkz-BU z3v~d`x~w@`E2QqCwN&5g5^D8d7@J3X`mHU#`T!hhcc=*w=sIK_-t0h6LFm2JfS2Jk zJ?BAZ&g9)v26|sKK@i3WCvlkiSGlT0^$>dt6WQzI;WW3$`r`{(d#=za|Jq={ihzJJ zcEnr%{NGs9F^7(73asg|?S-Dh^#2`R{j7~g!{b~!$17M>2vOLiijRr+v*sgv6a zz2CCSCMuwI+o|en*QQf5z)t5(Nhn05$eCOxr3{&j_F?^i1;O_{p2>zn zDFJ-6-{Zz@vyNm3dnia#4Gaw{lVc0$ zq^@oEOIHdZ(Rc-RmpE3t?w!ePwcBpwFe4aoZ&tvUO7RMXvs>QaAX$*`agPS_vXKt$ zTUh+1c;=4PTt{%Rcj8&bT}kbK*E>3d(@&S(OL!w%aMwxD1PCQFQWV!?A^F*yU+zKd)LoaxfOi&w)|aR5nURtNyx3`L-YqodSsJL1yK&VNHpuRtJ` z-43x1apjD7FTy}5z}UON;kEU29U8@+8@&eOBK5wF=WhI->)?4Uzj5h zLmK>(KEl9(qY8eTsCv}3r)m_J>{P%&9eAwgr=x|$`30b)5}2LIL3Xeqli7glPp)5| z6Dm5DfJ^G;8Ur*w*(!k3EPbn2-e~T1%XeRYoOPA7u|k>~t%Iz2}5NS1s0Xq{7c^*?WQAWHIZt z<2CNDEc$}6*1Q+E%Mp={ zcx|*1yGvh%HLHoyJ|m8fAm8|6~ z<@L|SIv9ZaO(G{)&JymIU)zY!6|ZU=doM`(VU&q|T!hEoi?dZ!BXYVP&wkzRs$qM4`J11%!AZJ;bxm&?REIEiRJt6?jVH z>iWfn1dpV=au>Rq-?=meCxMZ@kp7>9#wsV3nON2aU%D~O#pjJn^OqvfjZqK25Na*a`MB& zG+EK+MQyq_tleAEz+`F9=&zJ;S?`4kZq)ZYzg~1In_mi3XwpasvGkW!Bmp z5m!^}gEeghaiI3Z%lY43a!RiYNZiUO-f)V`NLh61eFT)ghxc1UTVQ`z_lLzVt1;i= zCuvR5iL8h48e$8epb8*qj-ix)sU@H+-50jvJ8O++q;NnlsNpYJTRfAbe*uJ)r$u|4 zb2|vHFk$7kEQ${DNEXJVqs=8ZJOuappY`>xT4!6(6V8A0CoUXZxa8e`rep8=~`GB(A9LuJKhj!iNcr_gv{T$rm zxts^^%u)t{6W&J`sh>|K(A}dLNhh+U3$(*_sv5#w4XH_Gryn=MsdnI~ory+LD1Iu3 zMx@rsL+dzIVidUl}EZ)9WSkRi(Ob=%Df z?MOdWxSv!O+^Rlg-*5X~{ro?<(U;;AWi?jhvpfNHU(MK^oLtFjf~9W|t!K_>chT3~ z%b)B8Hp=_HXMG%x2f1hH@d)(K?NAaOt@Wb>hY)(Co5nwGM%lj=1>NWiS1u;A6c5v{TuL8+HUnz2i&?ocqx@^_IRs@5x+p1Z9SQD- zT?Y=>-dw)IrFQOcx|%onjP869CEJ%n^&LJuiv77ee;-7KV>5eWXxCHeUx+|F15FAO zF9Y77Pmm;ZP_&U9g`ynnvatlJRsM>{eChYYB8_{Kk&oUreZ+xy-tHCZ|{%{s{G1qwqR;6oIgRg!Wr-3fO>)GI7*M7UP=ZBL%E6VmLtPcALDM8E2uk!c}P@V+7YVja%;M&N{Ap$x0L&Le4dzLIB$28 zJPaPSd%4t#YQ7&ma|0Kp_jgdw2k{K=Nbc5fNp)cUL?VVq)qZ%6a_2#m1J-s1g&Kwd zBK&`6AtX3t+wz@$zC&=i4R+#rg5IG>gHA9yh-N&J~7t_gOwHsL)4jAK@peQUTw;9-5fAJ{KL zc|(G;k=(vR3Qx4gfS1S)wxnrip|m@odHvQL1L9i2UEy8;E#7>%un*Sycu|6Rilc08 zkKYuT?y-kMd-SSe+;U-`!?&O#o_UeclEI} zg2d|M0@dp+O5H~)ZW1-Z zyQ}W{Y|eIn3ET~s9N{IxoU{3OCO@o;%uPul7h())8L(TGh;9*e6ekJbfv(Y#)2RAd z1}Bf@yOs-qG7kUp>yCdozoT2f;cNW!f54;MbZq#5pu6a$pAB~%uqB@c zm4JH?s_ytc%i;HK!lOB3*}WG%N#ijESyoQT`;U7sI1e9g4{{ZYvxeUlPQD*jGg9sv zfA*;AL7A8o--UUxaItCu7a7a)vi!N%+9lRv-VwjyZ%i3C+QQ10)aZwpwqbV&#WM z5t~JU>bw`k08U7~4}gQ1q`Zb*$;M%d$5%CBLFF=SP5#|S&`(N9_>VvIQX1J>@-rpD zCTYm7AdUDnjmkJQY;6gL^zB`qVy8X7Is`{a)y#xltT1?Pdvmp9J0UK((;JlWOR|rB z`GpdkXnnhLwV)I;*6Ta*GogppAB*K$Z}*NL%yk&Rk$oh8$WAtvKiYZ@*|A;arZ_CV zCwh@jwS3qVVR$^DgmF{w(ur(?_r*8jemKu3cNKznsLv+itsgy4W`8P2!R zaW*;QtfsEbD9-W4wl_Y^;Jkzx@Q#Kk}k`nIa0*%59>*PnFS}gR z?xaVN)}LE#{Nmb$Qr1;tiAnO67L)Xy)qA@Ecdu&jSrDhhiEZ@$J+a%5*3@23(gEbP z=JrA*TTtX#i~rp{0G((}z%h16d-OTl_cmWV;_hAB-`-wM)6SjNk&7aA3Cs8i7xLcr zkmo+^BfGM=tIn8+bfDkazT66&Rfb6&(w%E?h*0|;>&;zw9}`M{w(VEk6+WyLbfu6w zi~Eg@WE;H6bUw#oG&c2@f45`p!3alesfWiAu>$IkR1i-(SB*-4~H|0CB zg%dHIb3@rG2H-5QCo|boOP|eF?Z0ef$<-tKcT8mHI&rthwh}4BU>6+RX~G zALVji@WW$ZZpA$CG=L2tF$M~~+rH^tAE+e8QgqEJ0J1v{*|tq~H$E&msLGj|w|n_a zi8ypB*z)_{8K8qf53+$w7!VRNli9-x0##luMoxPa3*bCf4wNQZuE2Pi|9a*7$xo@Eg&@RDDaSpk*QteD>YZJD_amEzw5zg}kzLNw-Gd zeo+=DE=Eu+L)-hozd2^%1LEt3z{Z8&k?XKcnGN>7ALMd)UP{h(NIJ*WWbxxZ>f2oZp)vf^MhEu}XsNN6KfyGyzt zu_?{lSWdOzyp~9+w9mnaW6|2P%!%9l-o|}tw{5fIv5CUbIXe-*@C`yyeKNn~V)*;d zN~J$#6S;6?))gtWQwbo<(G)qn}zw){wP5=da}@sV5x@Wa6VjGL=}gg@9U{$NwK zH&^LMQEGJV4jL1tE(}MzlifrS+r8Vb4ry-44V~XoGmyKsKPN}X%1>aIyd(zwYgBGj z02qk|x;y?ZxgCl><~=S>?$R`mC3c|Zk^aqd&1b)HJD1Wn`B|@Wi=K!U>EBEROFLN& zrn9Z3{@fIoZo)*4X2}9Wpn-Sb|&YC`;>xu@@uXNndX!Wcq+gCL(hJj z_N|BSpkS8wgi1qtkJ$uQ!SFU09m$(hPP(9-&1VZ4#BCve!7GwCpc=Sv>EOwS;?#iazos0?Vdtwe&1p> zdwe(0k&!lB8Jv~NQ%;1Q+q!m75UFm$YYtnTcLp~vYaXG;$OHA&0Mpz^x;wtGhCSu% zf#Mvm#yzP)B1Z5dC{0s#N=bU&H)p2@hdeI<{8P&wr=!kt>&nh-ajo%25raxu@z~9~ zZBHR)IG(myd#)#LCuE5Z5XF2dDnmj1dlPIp+pky`RjcZ(MwR$eZmWVZpV&U15DwgE zPr9@7Vd@oFZ*~ONAdfGxgZJnnt6{Lp~iiv!&8uXaDTefKCNf^B#ZAD{n4!5yM&)ArEY zkMxY$qa+k$E>y%e$pM> z+Cy|;hZ7TR)mQYPN;9iEf-sfD#PTa!onO5Aimiz+ih`;fABqI~Gi=Rdf+&hIlBPvF zUGEgpJz8x9x2FzjZ2 z_?6@ax7tEFG%S?7F9d?OW9YM76;Vq{XLu(D!^;|zguA&dGINxb&Aq&-5E~pZO73R~ zBx(;d%a}dLp^0CCLELkG-_vhNu~Sw&SMWi%h`*K3IcDKNe)k&z5hez(HqQldB2Rma z99Bn9j%h>d<2@}+Z+OGj`riY)iG;W=>cTK7F@phYrOL~T8ZpXj7X=f#&r+wxqXxo6-=F_A6$3{B4!Y_iKE<264Q2yDOX4SASao(qPtSkse;LEyGVKSjRHXdqu}tNJF+ zC4I9L*8W`3UE^cKwYQF_qPRACWlB2WT=kKGUd8;aIXGa1IW%eg{%e5fi5#mzZ!axb z&+x9HfrFff5ns_~+M2B%-8=o~A%7-v!+J~NI1t6s!R z^H$@dMe}wv!6lK06cop@PmIQ)Jt_!3MK~+I{R2CZ3s@2c(FhHZoy^Vu@sIz~rk^WM zT1%JBl6)Q9ouvstg$TT@WVIiKr5ZP?}$4rQXD!*kxpvrpMV1<@dBJr@%qc)m<} zb1BNsFlULVING-4U5~>AUal5Fq-lAQ?23aijL9Z;}4+?$>^Q@5~zuiihAw^O8cn58yx`~k#XG8)Mr2)9>of6i#n91GC zDoQv_r}L0v_oGaYj-+azK>Iz2GG`Gmb)hSJz8udI0h!~q;`y=GkVt z+wgo~g-3r%KJe!qSD}X}S6eP^Y!(-5qc`HI9Uk6ejrj{E3rjig0*52!w7CKkoB)_5 zE!Hw9v^=;$+t%m5Ur|C9Jkw@P@jb}oB3NpEz92{B7V=}8u@;}I!bcW_kPr}w1S>hO zta~bNI%}`yQHid(-^w%E5km=MDrBfwnoi$__I9f!P*3i#{;0{G7DE?#Z4c%|Ffx(| zS+w0qF5u%s*vQ4e%VHpF0U;|Uo{xvUIJ~0Q9<#|T!qTHgI0MMgsqCz)w$~2l?vCwO z3h5)y7gXK~MRnBS#7rK$l3YAk)J!;Wyqcw~a9x{xB1Z!)U|FI?JXuw9IRhDGZMrH` z;3LT&v7D=ha9poITC-919xM@LuKu;Tn7!|#QzCBqH=idHB{IRHggCnwvRAse?f#3o-#Ow9hTyEoOtDW7KX( zKdBWGn{X#U)~e3h-CPs0CiEiA9J71TMy!LBoS(v5-}gzlwEIDxh^Gd6(L2}+53z>S z_^1wSMV84G9FmskMYi&Vf_A|1CGz8{7vjoeS!JtV2%IrSB$HxXwdRF_A2^Pfi`iPQ zyc089WUAgduHE=7!D-1BYkdwF(O^ZjZ}0(0cv5^)8(s)Pr?`d?Pd=Iy1qN%Z;98AQ z<^p;*N!^ryDi-eM7ZPERU?MwUdLVWr2%p+kuPk?&9&38RFM<3e3HU|O+|I^m)>x6iJ99z?ls>wnfOj&ACNnlC6FSS^Il<5~jTZQSi!57NIuzSEpVj>Q_bjNQOP|n8@uKTP(Al&DP z=&F!|9Lq0||4zQ(C|&lIBFS-Eivv5AJC9YO%$djSa0Ln(Oc+Mi0Y_daSf?Ka+R^NO zNm+{&sp&&I=8M4MDmDA@7|apCav#zYIT%58#^H8$*~#b)^Mn#8^L-G=bk-UfCm090 z2W+OQ1+E38i>%mejCqQ{Lm-_PQzYp7z;O{K$ZzC)1+O$?<%^y3=^TIuUhvNdB(QAU zSIp;Z(f9YYH{R&5OJ231<7Hc_j*ImDqjovB9+F_puH;u_=rZECTKy&rD&Y{XTYyQZ%6=C=wkyRO@(HRtQjq-|mbue3M{2<)L)e zYV8HF{OOOazv|gz6yXtRgY9tymbo3;NPEhzi17q*MfTvT<3%KM?)o$l8?~8qa95MW%R;Uk=oaed2Oh|FYI`p z2?)Eza=oRb5e;)9S~?yq5_~)98(a7qsqLn#I#`9#PVxL^y!S5dmdz&EjE+wLXyr?r z%Qg`BdQ}uHY~ELNJ6fJxsQq1aTtYx`3t7T-ErM5Dj2=A7ng?#Xl(X&|=ZY;=3n|m3 z_r9z0)!5~{i%%j7>9;F+)Cp@CTHLGcAqCk@(&n)mdnGFGn)lz;ji8ATCH@Q2)!aHEQQ zd95)3)ti1lfA7UL61@7r?^4$;ney0+L8fentz7Y9kUEIN*^Ux8Vr7&mmWQ}|(o5N@ z7vr?bK!_w)*wVh}?2d@8k8I70DPOBe$;b0=YhNt46sN8(TbIKWPSsD-djGyO{U?86 z8}jeRN30>?8$l?#u^Kl58(QOwfx@~K1cac`FQ&>!DIQ%V$`O)s4Px{Q+x%h>3wwHd zOW^k{FP2N9aQRy+u#|cQL<*8^$`L(G9W!lyv0U!I`YUUBG0-+LVa`-*Ru$SN+gGo( zIqWPF?b>r9KO;J2M}mQ3q$rg;9SnH~)N)EtbyD{A`)V9I#A`Vi)001@-ZiDuU2c6o zlKno_%T7aB)?aj#?z93YN7AuE!cJsW1+_*cW0Ns`aaD>QAcKyns!TkIyFDGQmWi9m zvJw&3jyU)~`il-bn&8V)7*_RIuF^<9!Vfzh=U|A-b;4V<^5~v?@%ul5QH?lDZt3AL zySWfT7cSDYpEBj9Wx;2>tL=&?Je#Xq??>=jLG8ljPmod1=XCKr92bdju9`*(OOM#% zBU{LWx9JzQ7$;ijHt&|n<0&o1Y@p-&Wg<4UxUq$R1)tGO2L!*|V_ zJc+w@iRFGh2Y~_m5|l3n>_$~~3O}X7P}|MeRDYX_&ky^h_$IgFi%TM$k&`BzaMIZi z7ZksLMz~;id;&#yMevgZj(y|g+{=>#-LE%XYF8X*0qMRs8zpTXAsHU{nqjsKtMpR2 zjRSps*rg*zTk%r4^tf@2*~*s!XJAO}QutLbCC>N=(Zg!5%VrYl@g+aC&Qd_a@dP`y zxz^i|*qm!)>+)BhnA!C&B^fn>f17QH?iJhQBWs8*W!WaRM{!~;Eak?R9O+?zQt5FU zUn*E(2Pw5T#pmLA%iV#>waw8r8dMB6wp0Tt2*=i}kIYIuvaFw)5;BZt@EmW>257<; zt;K&qA?q<{jpi~MpJP`S;FjB>Oz2F8W<^7s^|;KKWbuW7200A82EpDsy~@&bH@3LL zyGlUA+Dn1%Yj_{@`*vX^BKJQGZv7*K>_DH5#DaNhM^BXJ1XVr0$Vycw<3sE=3NvjJ zF|qg1EJKszj`xwKrsP|kdMU84to5O$?Xa)&lZ4?vl5KG#BrncO1}Td=)P8TtgF-7i-gHhKX9wOAn~Bz9$Fyti#4Gd1 zSziPaOn>tBLK1EU09mvco+gj^uJQcd<#P<=6Uw5w63msW;$Owq1Y zGr_6}O9)a4$4q2#LZYo|iDj)KDal86^9OxvpIm`BXgXSUjZElm&g@~W(PIk1SRu^w9+y(`)g=2x)Br)=HJK(AovcAv+#V;Yt_ronDLo-a?>X!>f`_Zd!Sj)@hZZ*7Nt=YZg z>ktx)=vCW%orb0GYxjvRO%vwMI&w{kJlRjYS#C||%Z~4P=yl+&7X6{2m0Mgf@$HW? z=^a0Sg!mA0VKB-v7Kl#rh>yE8{p~-o`pcoV9aXRq7J}fuBN+GRjCP}dw8;dniEB9Z zvcDL9LyG-G9H_%N7Kjpzz_Gk&M-on+0mvwXmG9b7AHPrc_L%=IBM&=X^^E;nxqEU+ zk>Ke>^y1PGbQCb7CzqAEiS)-d?SKy9Cxz3{%MWE{(w4{;D#$XppVKy*b5!6&oAX~} zwUc0QJ~|fxCp4g)Kj$?-!|?jHkb5Lmn!DmJMkA-DBol+0i837_tXS#jGh3?aiIWf# z_{=URFiLV}SMnG8KWtZhjWDz#YC`Va_MLXUS}zQ|082t&Ss7sGAE4^pn6sNv&qI5- zld`8pyyfeKjUM>ywx3%vkl39ZWz(Z}H~%In=DmQt3XBW6bAQ?iC9LebQ?yI|`1#;R(Ax)QRKK|w#GZ!hUn4O zb|eLYD;}2n9R*K3^2jT15Sh(?EKTo0nX|zg;s3hB8af<*u^C`zJ!vzxF$Vz&;mTl* z9f{9KV2N!!Z11M*ua10qbMzr|8Kq|V5S7W6DAPlsjkw6c-I zx{@j69$HSBd+PXIugVm##Z{CyqD&I5FR7>S zVmEU-D3yqyWVfP@c2E8XsNwClm!bA2n0jc9?^F<6#8~fcM^KvzN!{Ch!0!1v$OZa- zEOJpkmuc!L#ka~HMH=Icuf&`9P`nkLG*+D`NJq&XZJS~S*j^#r+Z;9A zInSK@pqIs$e=#zv3`g7rXi1#rVK)^1d=NidwAy|vIUd`$OmxQnKhjoK21QRxqosSF?btWX@ zli33jiO9`56@Y11b=)LHJ?z6WCwMkH5;Y3l%f?7&cQm1FLzZ{U_jjn9%E$e~K*?)< z!ar=2qW zB`OanCfeXURJQ2?cXA=iBotzaBr9C>!wwDEI|vPTDXU2D#qdJ#OSuNZ>2}$#78*Yy z{@{RdCBKdh<7Ri&CykLL#MHHbQBjZIv+KD~+xc|Ee~{}CIl)a|oTcd(n(fvrffb0) zUYGgO?d+c&*kgBc^qsx3g0Q;@Z(-s59Nvqv60Dm!d=7HBAE3x>XU-nva$%fC5C}|B z?bU)3Ff|=V?pjoVw&K--P!x^FR=ygf!`@-scWhPu+{w1#f9$QLwL(MT%78Vm#;xn? z+K1T^B+J@YA0i>Rlx%|}?>#mZfvBy2HHeqT`xN@ZHbe_#NW9DqRf|(a@=o{`+Zbg+ z*2*_6#dBkh%q1Kk;@g;Qs#B}T(X9X-I-oMH*uZB6k}90EBn zUX4ev*nkx2ALmKIkXvFXnbAoeAn&|ZZz~@#LTt5FAWPxn&suv=U7LtW^~GRaM#Di>GD^0&u4q+ z(5rU;)j&mU`{fXhW~OQ@C9NXxUNn2nZ-So6Gld{nq}R`(GFH5njDrOB{+z6gGCd&p zib8drvQ-^VFOd~l~>*Je* zF_0p!m3<$B9wglKYvtDLY_W~6Ii@?n^=z!bQKo6P7v2B;N?^g3YmEAD!M9-_c569o z$+`zs;Z2ZrJ@d3J<*Fz`1pJq*)Go&vzyDH=LQxZ^INY4KkAmN+*&<|XNzqNo(*=p8tsEZMv6wd^RORx zE1>MIA1D%D6ei(bR7rO!#lpHDlTs|Kv-TirlpOU14S#6X8Rti2Y!T_L=!~P@Tpop$ zorztlsQ<+P;Dp=V%vwK-?apvbRuc&2N5zNSb>#X(X+=eGg{**!Q z7Zn=5 z9;ZGYi#=M3 z6W2(T4cBvPi^&%O=<{=ZbR25$rjLnrqqwtMKIRykb-NnrD`ur9A4HK`f;C-{0^SYy zU@q&-rueDU4c%X z8tp|u+7)POTh}NcHFSNy9`k@l{$KuM8y()I=}muTjnyEZN==C{@Jrj2gKWp1!#3w2 zm;`WLwsa*fOnPb&YDt!&Te|}9^$hn}lTQXWvUKN3a^ zR?|KOFE6Ta(MEEjT{TShte#!8jN61)FHY0gofUEBWLLf#L_0#uIcif5r8A?4a|BT} zTtR6Mj^w;rM~8Py@_RHZaSXmLh>xHpu~P7V+}h(&L&_S+Qu%n>iKu}Y5lq<0s*g_# zVIoV}>4g2;Ih%>^BC@6Rf(Wu%|Bg`ZjLrGFA3y>yA62D0Rniz4x17sW8AGU!vbNfO zavHGga?9{rvsl4_e?H^Ls_;^j5mttJ5}BN~rRYH_UJ)}q?S;%aWtX!y(q)&$pSCMi zuablmY-(4dj5~`AUUce*cCE_^3jNZvLRJrYExrW2qhH(gTr)@;^6(`L1?v_EOb&tk z25v^*sPd8QiirEhZh6_#GmMFY+ZLjY{RmVTAy_Z zTV7Yj$qfO3l!Xcc4hYuZAR@$4#_)~)i&cXXu}1&JA}j*xrf6Lhf}9{lCEr~2Sn~aG zKF-*dZr|63mL8{KUtNB(HT&V3$>vgiL`^6k{>JcN|81ZT@E(<$4j5Wly+*kI=u6KXmD4RV;RvAv{ z0|~$INtUkNjh^+IQir7#@&ZLwBjEs8YAxE$YO!Q_suP=wbt_?j1+I6YFT0%u(gEl) z-iiLi2AvY2#1h~2wUTwN-OF{ry#~!s)W&j1D*zeynLVgB`%|notfw664L`CKJq1e~ z5H+~6CpMf|>54SEswXf^t(TZ7oW#{V59yYkT54jY=$CIOBEM z%P)ufhv0l{eYB8i#81>vnR%I1{>i zGRoW((PP(4Rq%?r*FL;whXbG@)dC;|YjhW$BR$Ko91Jp!J~^41M{qk9%QlM`;i4UP zkh%^^lXt?;A-5XcvHuMvSF>qvvNVmFx6x)2aCiZ!R%F33>$S0$eBS2#?{5!7Cd@}& z=_Ws7u@xchTrTzqmQAN^A?E^5$1KDyan8=Oaf?-BNo1u$nz2h!R(x%!EibtxU-19! zIlJsb6CrTWu2f*o$?yr5c+sv#nc^-@aXb`L>snq7-E)cCu2&6(W5PT(%5hV)_EWfncb?&6v0vNH64uD6=l!ZynZ-&d$g!4q+>A6`|bCJcU|H*c4` z*Bfeqo~j_Vw$X=VKR-o8-ss(vsA7k0N?=+@(Vk z{UwYdZed;TcTfGX%QNMjRihl%zZ8ugYMcfI>FZg9+=%FszcGAIWYcr>W=X zotcDcq88{D1joUcj6Ox!>PAWQHx*?%lBjgJwBOj_T-&}uOcjn~w>X;-8-ad~#?~%N zyIiiY9$47{AFKKl-Q$t+k0+$hfO<|;72HktBeVo3t9^TDi|VY-bQ}U+*6)5}Gu5dr zCO_bh&$)Nx%eL7(9@|=N&Szfh?E`<+Y~By_kN&=$i=kywfn@-^-}|-@Wy*uCb@_&M zS**qLN>I@;5TsqI;FH_IGG@q@a=FM~jMtz;0Y?0}UCH&A9pQvs&BNmM19mNcV-f&v zwd>hY@sqkyP4p8uJm=>8Y~0Ltl8dmMw-VNot^-A1aQkn;>Ni`m!#lYRBrvzTerx*3 zOM5RN-7k{R?)xM*5%kp_I8?_Q?dVwH71#9!uGkB? z-!m>9@OtkXB`l`p8=^PKrR0wIE=aziH!vV!0H^g|+nAGyQ-pjRkQIunvA5jo$Dytf zN(|Yi-Z&0y6%JN)+w8z&Rv><3OMdPf94~R8w zV%}~oQFa>S&RbVf|F!x!C^AGFv}Fwryo8j*mbCWXf}+Rwax**f`_kF(-exMYq3iVT zMUGO-QGfeIQ7vk!{R#U%EI!3J3}!)JSC4dDHWC}f8H6*b=Q0*+8`xc&h+g&lY0nj@ zeX<(q<05#L`^=O>0{%wHTtTW-pz>QjZAW~Z;gLd`9<6G8%xZXegJZqrJ$wA=C+&EA zmR%icSB~S1orp5Ww4pT$;7P_tB&})RSM*Qrv_6x)loQ~xq*(_OI( z`tq(_jxx~|TINrGEd=jxx1$7Effqa$u6+E9cQ5OM4K0)Y(o^pFmd`?$eJ zU77|@HuNR01*>5j`wGr$-f4|qUhLorqtX}W1q*51Z8rCn11){lmV~W@n;){R-UBN8 zroQk05JorAmLsp_|5|fj!K+wtk;$PYK$if>jbUc3H75EVh{O3?(Rzw@(iSUUo;}?o zUN`q)(dj&6?a^|Xn;%PBqO9b^cx8L7Gr?DkGv3QuUtnlN^Ti#JOvecr#@+G(SK7?M zgd7t}8;K<(%R2;k-PdNUUbBhpyjw1$LpB*bDS{8&r#MlY$|HhpG0zL4ne4i$7qN`* zRL3hn`iUKlVPyLKfgMYz*!_38LYU;+@tj(pIO`Mn>uF8wq=Pkt&)sx174rov3=a9) zW~%N!g)Q^fYSxkY!W`wRvCo27LPtd~^SgH~r@vEbnCrff zkSLGYjocPwwFI}b+&RN1?{!^jFEb~iyJ zo-_)>d%45pxGhm}?q@&5V%cR6@)sv=x7n`5C{h!aebB#UF+>Ti*p>W?#NQ%`-?f!d zrr-=9f+dv4wrZCn6SqgvJ99oZVD+-v+Wi?eV9l;_?MM4Ajh+C(a^YibM4q?#UH(`~ihs`k z-aCv%??SXkPReP8Pvw1B%+mm~fv?J?ob?WfZd>x>{`BK^c~|EAkqtIpjWWy!tz4

_gqpVyRK>w2W@>*UqUTcC`u{Sd@sq4FfZSRvQtN0s)I5(Dc3(kX$QiSs%*$S2Z9BzBnkJ5_&QabN)qP3GMwBZ&`NCBERi-K+NF zBNTl1{aB86y>1U;@H5CcxvwSgm`Kx)&H598D3-FL1J8uFBECp@C4mxou4B$C`^yF$ zWEE|d|J|)1GpqaK4&jj{$B#ArNu^-e!Ia6=W^4O{4uRCWXzThN4Z1W#Xz+*HyY>AA zVf}*++u-9soj-03(QlC|G8>|q*v2SRPYB1ceE85B`zur-DsGc^Hco)Js4P~-^#dQ; zmSr6%aR6yYY-^OsiJZwJvH!}N`eVcKhm`fv=m7eS=9rZny2!U)b)^{iA{VfybNE62 zeb&|=2nCI?wR_V6$W??b9oZDIp$-~!vvqoPg3gf*u$KEJ-j!`&KHSpdgE6hEtz`G0H#1q*%%KKn|T{#ZMqL(S6vb^YaNl137 zKY4e8TFRzxDH?LFxvQX2F;~mh6B*Dl$@!JM7mqz^SM%?kgud>tL=dB(B-w&q&ykQB zfoaLp?2{-2HU7(aTZG9F+f-oaaT3RP(QapzF=^54j*s{4VUW$;>>P)uFcJ;y+-@L2Dzax2yjI1ZA8@U*Rq?gVTsAhFyi{#a<;@e z8ymOw{E5j2Mv5Ii0TL}K#&Bo;+PmY~$F1huFggJH-0wZhd4o{qXdMxG5Bks^gJ{H? zA{3SbQaw8o>tz!IDONAhiL}L&0V&~@iXA`sgKVl=Dwj`uEJG2v0BLd*f$eba@wR>> zVnk*?;-heIKVXlKMwdn6$4bix|CrYxkBFRQjt`^)0)gr-No7yuYRI~nK$Sk@nh1;O zT-a@(p_vL?od;Qyd}^~%R^s24xATU5=(SY-jAIxA^l8yA@yM)GfrX zyvMhF9lYZHFCrBB5H()?w#KTo&Wt>?e586Fnu}2^Msol@X@#YhKfFu2$J&1#; z&~&!juL4B#irsO{l!H$<7j=15d1zqf{L)plS6SH;~0 z=aodyjX4q2M3FaO)9!*FM~3%vjyG2@DX0b_J~n2HS6G_^nO7O}oP= z-AV8gu#04ezIH_4v6gJ%?z05bk1-UkT7j@+N8GkM?#~!nOa$I_YmZum&SZB)ZFWQ% z^M|q!Bm0hZ?shC#NG?{c1nITz%2W;2+6sj|TYV;xo-+ za8QT|P4X){R!vzs{Z8YHcg~JSnV5?+pd*QzPE_@zlNTZRg`G?gQK+HX0`cTd%BXxM zmz?}AgN4AE&5~{`#gKaR+FaFZy3Zrf$TPoB)tM*-}JQ~7%Ab&Zh7^d{!tM0?W#Z0LozMdJKo6!d`gaONWkyrS$OQH zb}x2L{hwv$U>WoXSN!Vz1ZEr9;=%6VCB{njC)*RZh(`=T7{Q$OIFV9f%B|d!WW!el zHIt^#A_bHPorlS9R!42(U2`cUOhHn!r=YGETc5PGdkO~Gc>tL(Y1^an8N?$} zI(@|5Tfe8k2B%+`+6~@-F=V&UuqQCzaJNiPB_rIpr|jcv8?7+`Wd}t;$!KL$G#6v} zfo;x-Dmtl{x-HR!1?EL@M7TBP=8Q4dRCIDdkxlXC3=D__3TNR}i;225`!K^`H6F8; zgc9DohNy@0(dq+1R-jBqx=#=$YfET^tCX0c-IrZzwF&D8(8&5$XnD5OIv;|otq{pz zwd}dLFS{R5|DM2Kz3CY=SdAr%F(}%RtWr>~6p?jb_yMHl3MDv^E$}aZdPJq|b6lE+ zJ>G9q**?5hkO3$3V`Ppxvd8gYpb*(CIsPtaXU>jRvm@v%OZh2?iUU;NZ&9Xchls@q&PsiRRu z)8yZt#L)48oy#=_NxJxc(H5di7A!Lb)6dwVH^LH8d{0gX%H#tk`XOedOHrn?BT{Ms zp5t<^(d**_cEuqb$5r}jmNyZG>{`qm>RQYehpn)Jh+VGdp6nzXZ7*EVjhx6cV}qo{ zy16H1785y7T;fSC?L5`>NV60HYCte?Q^inH`81J=h z7SGTJ!mkwwc?~KbW#ebVUWN zIockKPDL6k&DY08O!yH`v@5hma=lQBVwC4yA~N9Vo5&?9s(EY*5MxdbmKO?RwKM$h z-C>*Z0>VSeEF0eAurH#5wb_yU81_zfG*(rfA&Yh_e(C%mCH96()5!cz45p}shmfE- z@dlk740MVFW@v~!e>NS94ka*?;1;DXYHd!?Y_;g;i-eLW7&ezcVb)G6A`E4635e&0 zayyzR`I@|bB{yA2(3-|=aj-mdLj!gxp(#eO_~hG?*ZcOLLmpqwVTu-=g}D-UMmd8~ zzCfNy$t}nCtFcEe;w~yk?x09MrQSJh*AhS?x)0j*=qJzOqNt?uEZ)e|E>E(X)fH9X z_u?msLyj_4-|}+=Mj0;tbhi^y*@BB?z2l4eXyec9Zmfo1aW@jYmn#HXRQ_N7wfloX z$UHtGF6cp3<$M@apE29(IHSPI@vcHu*^0dd(`c93j+J`@GwAX_o2=R!L=kQ-7|{^e ze{YPX#73jG##`&cz_PFPUuw&lSeJb%(g%8m^#OxtnN&7c3BJX3ZSYPX5B8;DuOmrO zan2qr*v2S>h8L1t`UE1Wv8qS@JkZ?*kvdT()~?9V5k_uyK)Qx{Sz`{vmc0el@CFhC zH-YRXg#>R{)82wex_c43#N4EzB@Dw}$A@AHAd(7cDN9f}Yf@sZxiC07aW~rXCpt{f zUwia*7OLz$>&QXT8bfJVrw`iP3tn;9U}NA$1>nX&P-AUe~ceNU-%FL-M_Elese(a40G@@(#ZPHkEB3Cvv_W&RHO> z#C0;i{Sr$RJGwWBQppD8jY(8FpVqD#pFj;%b6vUT#2V{7t%g$xPB5c>4e8(32 z^P`QMY%xC5M|mv?KWUd@)(SuIl`TaLNZukFA_Rn8_L)4j!>;_Gy$FB2CHIDu=oz`@ zuryLrUGu$_hP!Up6DZd83+|F^+iy5fO-SZ$27D+kZ#gXTA;xg(cHYiyJUjC2xs!`l z7qYsWXRyX|an|ltqj-{ zN;0lSFgyge)tE9*EFOT?FcddrC)3?njVg()URw#J6e_hb$01JKZ){W4bIGCf6~%`Z zDsFQ%1Zak~IAz!thiYkh8-|Lc9*Zh$EjH&O;4WH|cfT~fzn3?xIcpRE#3d0{&O1@` z6LD*;xw^`w#T!deTY#uERQMtJTBF*Yw~p)&a#P;@(y?~>TK_Qt&T3v287B5F{x_PH zx(#}j-l1_Dsd{i#XECbt&p#zS(?l+`)Ky4%CUeH5e}a{m%GQvqx>oW|J3Qp?@LoIO zf9>3EN28uUp)$&ww`18BXI_|5{cFd4IFF{xKA))CNt#kr;C6B-k!|tXf?&D8G5WOF zX~?2sI?meOE}O}<9Fbb6*hJ6f1mxvPbnN`OUp81aALB{Ntc_Q|w{yO5B_YznP&|2< zZ1wBlFXkkmZ9W1F<(axP^!-m|Ayv#e|406{rR=RdWS-&ZvTuH0o#^Pjc4a6Ct$r9% zcxMm@BGiq=6_`7|9{>8udjgJ37nyn+J4ZMrVmPoeJ>7;$X>{f@8nu_KWmUf6CaQ` zm>9Rf2-*FFdaYm3?-E7*ymYqj>A9B=Gt>+oN@hKF~-0d)FQ zJ09tH#A-P=0D0f~z0+n^XRv}$SydI-;XNCP#V-JXi;056+Df5c&usz!qnP0Yr03a$ z55Vi!HHx8R^jZ?D&;wMsP5HjylZAYSBXBrClAvMHj%52H3_UuWN{5l&t`acdSk#s{ zdBKin?LEpIbYi%?LoL6ulUXja0-Mg|6=BHSn2F7J-KU?l+59S_DQW*93g#-fWil`I z&ncVt7UTVPE@uFrElv1>A4T{ah_1y1d3#Cy!G2xJqxQ!Qwv=B73iSs@z8pg#rIDpC z&lOET*zhtJhVC$>S#U^@)uni*x3t<#M2+{a1Pd@4K6=8Ja zh+`p_1eDyMTm+>yX!KzffVJB;B@D37wmEzY>w6Bhin}qr5`p zrBlRbR>Nb^~77zWxmfB%ofRS3C9f{8p28is(WMg-%}kHkeBB@79iez8?K=N*#fSv>VZHg<-zXpCK^ zyy*3I5?UL<%OzjGp4u?mvgA-GECLT55(YN(a;zCo$CUt5mPREkDah<yCz8 zi7WX&j$g0mMebL+)b;rj^Iv-2@OrtakN))M*6@0wjE7+vWmI?GHaeuz-hv$)vtB_3 zC9@8bR)+UYF}#)q%ML_5pV{Ud4@Mjuv*q=Ix8#IoTYVyrzi3T90J!ZO2evt&kM)k@laU{H2MU|? z#ed|HjW(4#j^P7 zS{EU2N>1LPtCSDyzAwvAFUBqpa>tn${-mQxYJ6Uf-B)0s6{CK2-Cn0)Hm0W_ zsVhfg??}L*NZ)1eR*e=6vbV)nM`sdg>vo6%Uo+~bMGqFPGcu9^og!!jc%)XgHrsGc zls;A`XkE04$5^zPYO4(x+FXQ0*${0cZ}G58EDuOo2p@c64G!h2Kemnj zSKE`;IGSWnvuzp;BOx@^GJEbDS8$rm)$+)KLfIAHu`RJYexmd^$x1+SpsijTo=SXF zojh1b(0#O@{MeeaFHY~f;4>WOtR-8NAjtk8wnqO>LeM|+b`qj(wBUF*ue0{if^5NK z)=_o+A+kOs);!*wqvZ*}43?C8wW`95j)>ANT7Q(uQbRTiIH22Lu1F7Zve7{Gy?eWP z4eD8)ZJRHY%_lbD05M@SI5z3O3Jt$tQ`ISYm~Yh?iNm9T{E+61_Xs5)@xEo*T_>DM z(Y+k4+7qmSO_Z>%AW}Q#6K}-C>3CF1*H`EoVI*jc6WJy9QpIKyEORnv_e`rz$8^Hz z4MO=Hn~5?BOELb2t3z$)`}Y?OHs{mhZp!juel*mA#0Vv>1utfaW^^uBh88n{^yghFH2sXXxwtixg6sw=+Fn{tY5GzzKu)MPw?Dc^$9$S542tL zUUu~Ewd-Eo&uectvi{h}sNM8BkN)g2yXAGhJ;5&D9u54rkIZ0xiSJZX*EN+r`#W|w znj9lw!aeWl@BaA5c0aaR=kqIjkf#K@RBSwL%<+jXoutmtzqTUET#c!?nuds zz*da~eIg46kofAcU}d(0#n&FT{YCdQivMy#D z9|dk{7z?_j&>QJhhPRz<{J|G@t>?jnA8X8hTu52OBAcR2-t!-FFy9BqY>qPUgzhZr zqjR<;d#w_>$reOW_qXOW$B6&l6xE8ilWs6bMMS^_YtBv!E$n1^OVk!gu2&$k2r1l7 zF2$Gx0&V__3;zWs#_eN4;33K#GuIyLo*uk(YGZ*BkZ~c0 z?a!aTl%(7SV=(!2T(ps~IAgL`^t{jYn28RKoQBTB?@c7U*eW1vf4HciNY^AzOU1Qh|o~(MUJid7xr9+tZBR#MSH;$QHnx0`|-`i}S_FoR$od3;@-)HmH zid-xQRDn3o^W>aam9J9@}Ksd<7xs?RpNNP$jG&v4ZSIt{v~}g5C6nL&IR#TRCT_ zJfgea_I?J&y6jGV=_AY~ITd1tg`te4)aij=-McAk7rkOt&1{oMj|j3XnK8B_KkFXXcgvdLp`oS6OP*W(4jYLsH&Bhrr?8( z)ev$I(i=2qja9u*$`R+_+omY<$rM~;n|%gLN{vF!j4kzoL5p*pZH@X`ZbZo319WtC z$AZuxD8`rlYt6na&vJoU>M3#>#~46Nb!)w#)}|+{tzPi=HcpwfM`z3T({E_ULr{y2 zoZQwsAC?ImsMRm6mLT$UpJUZzJ=I^2SD~cH#o;1%xm*1INVF*I6us#5)T7J;fr+Pn z*CqnCg2;W3IC1ihm|OZ875dq9E5 z97fK}Iqu@|df;0EGNU!dk=++v_~w@$HsYof zb5~o6GIwiLe6kC6IhR2)RY)S(p{ivez(J$L^@`Q^UGkullIeYJ*L=}hcl^q(=Q5%G zCAtJ0QtrYI2rI0I^7?EW9yx&7G>$vBVl1KTZ(=`hSEtYetE1_TFL$E0%M=xGo0JSc zgskKrm!RPTyXOllb`l2QzHb@c%$w{%0*Ye0I2PD=ARa#d+*bH6v9uh8mE(?v)T#20 z3AL{pj{`A`gN&@*S9=rT39(Ai#A#jm4xF~N<3T5V`@h?|@wgF?`~x`dug`D9Jl`{8>aK(>Px6MafoUn-WU7$%#zZ?Bk3>oZFW1f;CCM zU|Yw78bBaN6q=^|NqVJG%!OiVNPj?qE&fY^u^{kTb0S^!;`iJZAf*|P%yill3QR(X z_D*b_tOQ$$X9ei-qIG(=*z(BaqE^l9qosEtzdmEh`=iWrWeeLZ+Mq*P(B8!#*~ob2 zp02n>e^|uTp^|WkO}&DrU!IzSHc{;zc8oTe&_PllmxWe2Wm5?cOYm8Ab2wLNfV@&L zJ~@FQr9L`39%_$baHpst?b!fki*CS>IpM-q>8$lFjXWH5aj1P8OewG6he+2&z&% zvjuM?3!7eBbXdw|VWdk@--#=iC@ekfgVR{@f7sw$Q3a|%yoXly zV}0T9-tiuWkRjOJoDJEe2YUD0z44$1Ug)hsHrL=-?#BdWAcbUrAdAE#%}JO1c|~Mr zwy!+DEjw(*zQ82ANQEwYvz7ZC-MECW*yGu%effTqZn?-0S7$9~6-W%=MvuI&+-L+4 zwsxOS=!cSfsd?7zE0|7dF&rG%`!66R!i{b4UqZ|fmJR+3pGu*QjZsrl(R0?AP!IKv zM_vREpQ;eSs;Ve_G9&zP!v*4t4HJB1JK8 z_F63g6O|=G29w%;pUU>_Ht5rRf&e+=?AeHac;-o~=faTLqD$T)6V(Jng6dq5Pus!7 zyqOA9KHKMI+jO|93=dFvBV4$08=jTH!qEg1dN|GjCNl4IreZ=x8 zQs&hd`7E-pdAsI){;E9c*JEuE4BL$g!1tk2Z)QjEpxDwCJJ(xLChau3H-x~a?6#Mc znEF^S4b`C|?&J=T08G}vclUk2>3 zm)pyykGRV#_B*O5!Z<+w8tavnQRW(@BKWmc`%?@~`beHTyj-)(>co zcnQr+HIUD-rGDes9h)IAk|*TeK}%CN4qU@w{%`Uohdoo&jsxAE*aQ%oQwNAjTSV`6rUtEqVW>Am4f~vIojKrYzq0-qGbttl{4)t*q{026oe^UNUj4rrM(=kPZkyE; z?uYDanb*8c_*hTqc#%dX;1-v7@mmNen6s&z3pOK?3~2cdM}K+(#kmk~p8g0%X-D=4 zrARU*@ZQn&4-zU}Nw&)-3i9cy6<%V+ zgd-oh!zmM#Oj}k?6vQCyw5{^mFr&jrxK>AP1*R!MUlCrfnFu09$DJ{W48y+&_H1o@ zah>`y^^C3a5n3c;TpxY9Z?`gcU<@}A2eeKU;xTK8Dq>Y7t(C>j#vHb|^k_LUgiE7$ z{2pdF2@a)nA#K?MGP_T$JqFY_wN*NkjwmDX3fm&F z{a4nRThZTD8?u^%M4%b;$^9{u$MA||0?1@Px50@DCFAx`LI;gxKV#$Fb*txek(f(< z{}otpqAJs7I=?mPTLO<$5Wv(#AmrX5{H}66Nz`_%*fq1UMHHM!O&)6 zbl8R2_$=#uoZMK|i#9h=F=TG?Qj|%)xKJ_BFy|%;hJNZ-wooneXFOcGUSg4CKd|U^ z2mx`a8snqlrh@;mE#>hU+jGD!`|(u#IJ@G_4i4MZTmW_n-*wbUFQ(;3TL}S%6?Qv!3bB+pBn3CqV$9o}+|>xIg8RC6 z!b*(Ty*xM|@I7`v*P39&dSI1v^?}#kt?(swATY~!Z^41<5jtfn4#d5F8t-_FZ4boZ z)XO3<1-hpLajrZ=GS9;5><5ZY;!1GsRm=(rYvLa#C;2IY(5Z2 z7(1$ACEea~ASos?oHLQF2MR7a^rH@IiuPI+$-ggxEt@NNU^nR#B&X}h#cw%K@Gspb^m^_p@k49fikmm#`|x~|O$H2N zRzygG@N6n}Kr;OEb~xrg&1-Wc=Z{4OI(>q#`>HQ!$E%}>Ud9EsFvY=&J=;Jfg!y5Vd2v&S0krti>^D2f|R9#pvmrt&SQfu`thE6E!%*Y?1T2YY&zy z*3*l8dtLSbayh3P#VfJ?V7a^a@Ysgf)o~Q3zXQoz;?~6tS8E(K-w@ z*B^6n?=^yq7uG}30tXKUvX@%V6ZVcA43xQDAvhST?)|c{;{>psh><;MuTy7}-UK^| z;a#Q4lsDOl!86H>4@VP6Ngh)9NOrnSnw_J*laD_7XLjrd2dvn#ga-3?E)mHO?SxMc zGm{ZJ8U2@RE}S?}o=tm?n5o&!!*yYsXMfPY-%??8GT}W0vzYVaT4wX!wzYRyUef0r z0uU7Py@fn43W(Zoi(Yf@a0v}|>EJ^l_2(RC+2s_#M9-qJSaR@2$ozFVz&iua`iWhs zfZ(Krd&||FALr#0oqd$i*Blfiv0$d_0T=ed`|L*aCmQ9V-K>7ifT(YIPck9u$11!K zjklwB7($WTgy|j*X$kJWB00G^6kE#u<>+>*Iz_M^PnnIC@Xiwu{V;X znJ->_I-)DyaDqW9r{CDhsz#)|!H-~7lvP%aaK5YG2x8;~QUMHf?IoM$8}YL8u9hUO zB3{>e=kQ3VsBB&S8W0@F+5ypYczKnTpIpaYje1krvL2Eto;o?o68Maim?sbM>Z$x z$a%pn$P7-Z)6O?s{UePv=sAoHCB%_{>+l;v=%KHZ zHuOlqR^A4nn&*x?<#E)}}0zJ=+f zSNa;Ha@EV0rg!v?jFSUX$w#hbz4BfPuCMHR1sBOpzUifB131N&bu}eW!{tP zjq4RCGEthCENK2OH`?S6>O#Fh>rO@4dk_xi+57XL$io30vS0M!E(g}X7wm{np$p@{ zqtVa}wG|T= zJQ&LRg`qE4rCLx#GIrVfc#=6?N%+M#a1j-kBkMP2!d`#L-FQZL$+tmZdG&rO2h`-z*)QP&`2=|#=A2a<_yK*L4m!MRpaaLe#5D& zUNyU$I|H7SbMU<%96$&xHrgX*-B0j25K?JM?)4yBBY778Xouoz5o^{~_z#6=++`~d zIYLp&WkvmZ7*%`Kp(LAx+4Zh|W2+C9E4U4cos?V(!?q@B>0$a>1Q!Ql?I9!OB4vRDLiWIng`QA3uY=WIju3*#tjI21ULfI~F}QQdebaG)3k^e&D0yH9R6D5(z> zRMXEYZBA&)S}1$KDcf?Wpe)gAh!<^ZwkD#HbfZnnrYYf-4sm9=#nHtTTay%+W(6fm5DK0K924iy~qt7bdu5NXS`e8=Lvl$iGgXsN#?{W~>BNd4jwU(1F;UC=%W|9?X!)(Qy@dT3-)osRCzFDrynCwhfz3I5bO`;-fGm)p} zZS|W4|Bx;Z-SV0@jmml5n~sD0u>^x)$+bR5am^CfMg)Yo z=ndI1fl>B=_^ctCYZKTqM7kGkV@_xUPjhR`8m@;Gb77m_416OC8BpBjsKbj}1lJ0jX}hS9!4ezEPfXyK}HwfMmLvy5M465-Nc zw80#;grO69@wMWlAP%y+Z{pH)H#GM|v`?D`q7vDpF94_gJ)5d34-DJkH$%%Kn>RQ} zWY8rU&5_)T9m7Z?jz*UPQ05zoX>crSv`9a2+;`^Nzu}?#$B3s+RI99POw!nku~NRE zlfEE35#Ux6#k`f}6^k>Iy><^^u`(MqLfppa=WQ;_3X=eJ%ricp@F_?w3opU+xrBgo zfEb)(s4=7gT6i-b3dIT115faTWf3YCeX*9NJKnHM@%LXZ9krzx#jkv7 z!IeCvT|)=w;POg;`eo+A)g@F|-}hIh+EfmA`fjA@Qza(k|liv?|I<1O)0y zPrmxsg$lqdw@?9VeqHX$(`Ixbd)`(1Nj_>m3{z&o%b5- z85+X|{W>Ki{avt(qqgzap-PD<;=V}0f%H-1V~a^+LkEugh)E8 z3$``NWRCfzUQ!TUlTT)8`d58&gleZmsre!@{vuS1PZT_W7TpG`B~`S( z*un2`;;|9$2fr=5!x>?Wf2~UhS$;Lj zTtHGHnAtT4YiW90i(PlH zL|M9G(h^0Pc3t90q8Zw~+(6{2&I{g;LtDOcHe`&T3-J|)P*JrXI1qiqcsJRU^B_|3 zoU`BYUrW>5CGuY}71x(@N3zc&@EmRu)R2Nc+hAe{dFDaTduz!e$8nyHZXb&Ws{*&jp|C>Guer_9$T zxHSba9kM^1a@<9Pn|0Z`d0d_tf0WbYbv$M36OyssUG~QT$r+-LoVE=WB=d!Y{D?F2 zC)xPP8nM6|a_+J^9jK2NT>r?FqhTk<^g=1Bk_>(~X7!q~)?ksR3-%KS^zm)~&>C|< zwnIpOe8u)t|NN^RO|~gM6Robql>Y>He(s;U4))sS0A4bU5ADyrGNz7^9ToGl zz+?A<5d(skrsC1rgTPPA-QnHr+-tA;UmIVuPXFuBpIUd$vt*{MIWVo@n!bb;$Q1wp zLdE){tlSZmkyqzbz6))5#D^ayYU^KTMJ(g+|FEOgN-67z1b(yjH@+$a3wz3r z<+yz?%l1!mji7Q|NArapPvlYjhy6S)*U$JW zRCyiW%dwygC0d4MwApCwu_%gIkW(sB`~9db%g+zW6G_TN@9|rVN%`NReU$6WF&6bS zA8iDMgxuKJ*r%~c5*y$;=c`WOo-RA@N4%-k76J;zygUzNpBLXVy(M4e&OD{~#fy3U z`Z4Dkv1 z-N-$F_LPX>Vf6Ta@3WQ=Phn1O24|<7>)nu#6*3(7~nU{ zzZVI>f6Wcs-gN+Xbm`Rho^OH!0fF; zX}q$*e&?;A0_q?K+G4S~wie)p}U8pL91Me&!;GUy&bKw3ciJ8yme``Y-kFzXk} zzv7XU={c8TUr*_ug{yWx0(pHgu4$PROs$q1(vR^!xsU7nSE_&0B@+cyh;+ z_6O04PZr@LoQcHWUqS6nL65KXs)SCmKdk!eC3Q@IEjt1hiH*Bxe-vP|wW8-;4g-4p z#MXN+kY)DAZw2cF>50WnG!eCUih(^2o|x`Gc`M#T!u1Z+*yV<03#EG?;`k9?4&K2= z+Zf-SqLGmM1N%u1RvU+mX!P2o8rWk$je%2B@GuC(rW~gT^_}DXc~vICx8_F#3h(XP zzu`gsb03F=Cqj*Fc`Jz012E699e){L&k$I2&bDS>ASH41L^QG|{P?`kY)$zUlK2As zy#pS2D*Nlh#1kcSbJQu0)Ig)mJip9+EL5y!rHfRoNQtZQv9;u}#NC0=%Fn)45m~+X zd=B*-u-2RdVyjPVN9-7x8QXH-hdBh#zvX0&G}Pt4)}FJ2m7lkl0yc>wuwZ3d-?0u~ zDMSWi_G-c*!NZVs`Y;kPbVom&M9_5Yt?&PUgYJ6prN0&I*!rr&;n}LJzgjlA^@$TX zW4m)Gi32bfsE*ph$j;?ivLRo6uJK5oiefk#yZ~&;7dBcA@QL^X*7sAZ=OPyTR5bqk zsss5mo2VxAF&xc7|Ftyz=$~zFw8?4#s+XRe%pIXfI>e)}}DQ}&OdEJG=v8MY&_gVW9AK^{;T(Z8;G6T&O%K0E50{`$D$ zLjO%(oR?p*V^x<5FD3J!ckQ1hpk>*X zv47#?W8n%Ze=_=a&DL4m$};!Gzs#mQh!?<-pK=+eqs*yz>9Nk(DKF#d9QbfF$dkIHPXhLDh+2}{!3(^vN0JZXd?kOV8&ulJGz%+{6kNj_pA+Z##PTu$^ zMns=x-(7vsL?wX7xvT-*N`SfZxxhnYOsl4Z;<~(Gv-VYDfRdMQ%oejp0vqE=C#~Xz zTxoG^62)JNMN`sBkt@m#@8$}>jS)Ql3@j{|aw$i^wQ{&;)V|N-S~P2ytG+6=CXk7w z{8#Zs8VGr5xWA^Hg093UIkMs;{Of9Nl4h3JWOlXMT=Z4oL!i;D!6pADYP&hDA~Kw^ zYq8|V!2bU>H-nevyj_nkvtW;Zm$M;qsj$wC9Jfey9rAylGbF$Z;hS#r!`ngFVE&2OcHJI%+Z95?6Snc~pd1)PlD(hA%KU(p14|M{BEGFPCxbT4gk(3hZ!k85FVk=A2_OMsv%%KY#oC-_xUuJ0KS1a;H^{ur0na zdgSacy$=#dio4qyeH0;3oV_P<)*}NW)>I9m7fjUntUcx3VUk5+&|C|Wjg6sgvtQ<6 zL$ix;moZx62)G$zp=^C;neI1`Q_tpjPhz0i>f=ca-HV?_38b?lV4$f21arPWu(n(Z z!XAv-^9cnpUrQ>;_8fDW1KR(^+S`D7S#A5jgNTTT=yoF{BD!9W$L%q%x;<{U$1C}g z+dOKbZg;!w_O@PK%a&`cvtE4LZt0krnVFfHc_UW7EMGEPnVBuk!qygvkcbG8h=_;? ziHL}Zi2M1@F~*~vd%5=y=eF6he$O+XImaAx%sI#V&=q==k@pgna{py02{O=@Sd^HP z@`8-J9+$B=hZ!@9Y8MaWFGb{At^X?CmTy^oZUG#TdKrbJWzN1?V>Le{t)^Uc&d1}- zMKR#M>Px-5;>uBPEskS?xEriB29ox6zqNaJN{Lc$9XSuOXM4JkuG}87)pXkOz@2lF zaAsVkE903Y(PMCU#OT^;9}j<}leQ)SZe*5gqwxXh7RrH1=ARscGy@6HOQmr&N|cz?@w`Sjd(YnVNs!*;PB zUPy%3)RnBvl#$OT=qpNML))F`Y3%6cM%f$7w;#pyaqIN~ac5c8u8O<5`43dIQ=&9k z=3X6=T!V*vY)m;Xw%TF;p}_kFJCgH9$dUF)d3yRBSASnL!j4w+I#i0=V|cS;ISh(j z-3^#5in!Wq$Me)ucodi3iR>IUyhMCJcm27f7>E2^fdf0`Ahi)O+hC`?7uj^0OHD_g zI7RQY!OkRBFe<_t5BN)Id7ziG4*BE-lkHqUse~IIvYLTi?T3%Bn?aA3H2Eyj}=cqB zcLn4JVCMG}GL=?RsUoK2Sc+pNS=6*@_Apowxyai6z@}GF94>6Ll2pvdXO0mBnxi%| zdvr&Bo0ThHQXn(^IS;2YPoWV-)?pyoCB+Q%+1y`&(EAT%&U7uryr528B#=fHBxa{J z$ukGbiY<(;B^&H561$Ee4BT}61{eB$xlUtQnfn8HH9i_6EJ|P-R(jT^#fiuIHrS^J za=tnd0s3wHhia>5MISQ&Q*ZT-0|l-Y+fHLlNglB4fJIV_@9rkwogCA)G#ZtXnev8{ zlC;PrE%~Oz;<=&1CA-$dm6WNj#|&a^ufQr-rEW6oaH6ScB^_l~yeBkODK9yMPTBID zeYrKbdsgNHQY)3(cdH$NurAQ$8t?Ai^{_%~6Hkav?e>swGTcWC1HLZS4W~LZfa_zC zt~5_xsYu!Ia1m5yPqB^uUFgI&<<6ap4=i@An1GxB*cD>YY)cHZ2M*%2ZFSsI0Nl1k zV;zhK$^uZ1)yG*d-*#lrbxoztbmsVksv14 zS?-%Xxe>VLC1~%?6_Hc}@w^40?5!pNv(0zX`UJacwE?5)%>JhD8$473IhE)0GD?0q+N0(|!tevQy! z`?i8pY9C-FAnv>BnG8L$#=T+R$1^3b-S4YsKlqAa(->nv#`~h&i{} zRS9s_6@1ce_ED($d9U3P??v6hW8rHg>)923I)TL7-y6kaCbX$E^>w z&14e)xXTQSIo<&s=$T3BnF;HIX~lOy336>>jE-+EEXyRH$6tT*Z+2JAgT=!V>>U3T zpNIQ(P-h`~u0z6~o`4fnw#R0y55}&oRw9RL9?#6yD^{$qSuvGd$`eIaE$1Nw@ruo< z8gtRkP3j`gvKD#X`f>+Oqbywj>D@?rd(#$1*Brw|gDgV0!Zd34`5gb{Uw&ivSJWal zprl~a)dMOQP<}xBnENAyU6q9z}j#6Mgj`DTc**IHYja7<;%B#UPRL@>3*n%gtjRB!{LceW_ z7W+AI_kd5Tf7s@DCKI!`>9@J$xA;KKRF2x3(9LdG*<9CV+mZqr#GY(-dc<>%{ky~I zuYRTNj6N0gw3mfP-D0~EIwBPC!bA)`WzSdb-r`arLj?E03(;jA9GvxJ15rO89U))y zZU>IATHBkbg148D^DxS)a@$yMe3%KL#ONLHsWeKtA#25+T? zs@FKCGxkXif<7m;fu03TM9SDvr$SYdvt#}@Di_!wp92R8Jm-o02E74teSa+CsNdVE z7@>rhaoMR^2~jdXBtVh zq8{MK6>zSYQ0`*CMlR%(S+U|N05@_mo~6c=dRF$y`n0N$4x=bNKh2>B_Hgh@-S?7x z7UMW?qnlAzi!uOyzTOoKE~z9NSxGH%7=2NVtYE;BY{R~cXK{AIH@;$DRqar77jEoT z`?{K=>fT4<;bK_76s>h1X-6H0E&3+U8u@*ZQ9(Wh4fd_?m?_ijJ71V^6r*TTE=(s% zcl#kvJIaRsm{*lN1mwl_lM~c$M*hWqUVrv~=IjE(lhQqV&S?B=*VWlC)fAMcmZJe{ ztv%wZ9n>~}c-14ptdQtfAt+ZrQWzP0X*%tiN1XE!C$4&3CBCsJ-L8Ek7D}-U?I=&} zx<^7;f}bur-RpgjHfaDm?S>c5=>qNMN1P)QXCcW!nPzSY z7({!D)97PYI82$f2xcZT1iI4c zkHk%<(Ed&-@BF5l@kqIa=XbY<53Fo>Gjk4&H4{n2?;_D9=nLTFWcP2 z@;ox5%*%x(KTL68Ti{;|^wcUDx3uYKc%m0k?=ctu?7mo@hK6>#zZ&IWIZID+aW9Hz zMdQUcN_xKdku=5N_2E^y9;iO$3_|AIgO0S;r!asA2|R!OBdNJMiGJHvt%U2ARVHB4 z8iVZXF}F4)E@m{ef|{ij4B`OGfn+W5tdwJq^=kDl271PgvGzxbBy!KxyR9Q(gA-A3 z3`xeUD>u2DFm3rGVW=%jRWb>Q99IU=t}SM)qE?c2&xO_1K1FH&*zs$8{CA(ie7QCk z1d&<{l|7V%Q)C`ZLf#fqBd$wyF6&kHo7|B5y?D(yQ{_Q08+{TSmap^w zleVcEvje>ickE_ILM`suTf9fispOhYO3kEuw#`w5y$EF>GQO;9+XHO{VVpgx!@eUI zMyxi~GrUvOqz$$!!9<_1x6Dk|A6c58ukgPY-x5!-_d+~V@-sQ)C`P#_o^hBJfl~UM z-3e-OB9($+Z%°(|DldSj=EyMDzEKOIfhY8k#CZ)IkD8b?e^W=_) z?c;bxVRSlFUgA@BAv!@efcw3eUzJoTD3)3GPhyifr=hD(xyz&$bNcx-dXSFmpnc{& z$ac>XH*lvVj!tZh`o}(t9_X#=3LcrbcBzF|NOcF3cs+BF)sP* zQKQG(H#wrRA$2s6@r9(A+0`-@?7NBtmS9C8=sH)YzJtt%KO`gt%i8j9+K-8UaPI0j zEQLjl{8LVay;*z)s4eT)MFeYNJ`~IJOY|y_j6G~B1@zm7;6aEu@tfwV4TZD8mLnJ8;ggdetRHWWIaTVvO5$aDF!_n;&J?P0P;+jSe91u3=DSP>Et+x6KkDQa5mh7G~! z;4@0yv>W3CS5YHw;U;=hJabk|^hRx!4Izul)J&!u)~(KN$&b;M$-`Jx*#>P0hD=d? zcoN+jZIs3m4nY6fZ8-o}_33iP@3GrA1XtD8S!Z`_2+jwq5JJEz8ysKs4iG&I0#DS?xVS`tZ zT1~rRtdDeWW224R;5=n72d8$S+G6YO{)OZ&E}W)bbrsFEKW&1`=A3EBEJn|7Z&X&4?U0 z2;7&jqL5OmWOR>qe~hN1`K)A>7E5MY8|(w=Kt9?RN$iln4uQ}!in{QQe%$eypJejLT$3l)0h z$EwQ(N8$vbv7v!f<-4o->zdq~LxOP;*5wrZy@_)67)kxfO|A1-f^dWbyqA{)I&e7vy`R{hi|}se^W38`1@qt* zKZq{HE$#zM$;cmG?gQm%y?qqGD!(i4!|_Whls$H#ns|*}+>mlmRR-;oid-)T)yblX zUk|6m3H!9V^omI6@qM2ql+Mz!`#ujS_n~$<#fc{S(iat)ixp987sUHz5{bg~@rx1!hp1-<=Hp%LP=AEO201Z!Rx0sAQztE-zJtDmEVcxB;B9QQBrj8OMN&=9HF zqlJwjdfKjfG?)`MqVh_Rxx%h~)bCTd21^%sgnj3m{F->$2^AsZGufjCR{$m=MFuYa_#7o9}Lb=eJ%h6)KR@3k8rO`jYu@^n*l!iiFO;X00P z_U{z#K{>ly9t{R=c>~6DakhgVO%E9M|BB+1cdJi-Hsa}RF@4t!} z4PPoQ18Icj=A&9UH^TXL+Pp*tmqE8Ja7s`T=Y_uNi`(i8!?t55cxf|SHk+E{q zpvWki=%V29);M!Fs2)SHKrRlPO1rk(9*F+RCSq@s)6r@ZHqxKq(I~dOzMA5zvNy4N zUbM!XVkhikQ}n8YizjSpPKEFVqONjXTcstFKSirQ0sAM}+Enkc_834A;20qtrFFzJ z-vO@b>#EjJ1t&AsM5oE}Y+d3qin-2KIuciq_-(aAi|Pc9aELD|)UfUdVaDhqu z=vzUDTM}=rDj>5p2cC}#s#)ydHXj&g-nVRf^o-I-3^u6Gy zl%26Q+C_V9SA6FlB7S&2?=46_tvIL`Dwt(<5!3&w^<*C$y#2P@;h5Wm)8*b=5ky?f zR~Sg8PCbQEq1X3~`*Pr;Ts71=dxh4`4&|wU)|y!!e$?$63VkhXRdL2pH6*A5AwWi? zm+VM2e|+Ph7-fm+bGXPRx5|!I?~j!0<1qKsv3MqI^_JfH?Rd;g?uM`0iQL>oFPCgv z{dwHi<3&lZ&M6;)olm?~E>QRvuCjN00A=`-4j#o}r=7{|#BqVkk<6b16>NfCdiT?I z){%hbW4SiXxx|TFq#Eo!KaJ|^?EM^+&Dm1&z@2~8rF>yWZaG!-I9QS1>w_2#&s@$2 z-MxMGp-%%-0iHVkH@e&f_Hkl}R`Mcx1J3$Fj@YJ%5N&{q(Mk?X?%hqkyF3o+xtu~r5DqelPMzVHQ5)?}1wU&fd6UK0bN^lV=xP(>pzmpEd>y@Pgc ztxHK*;wqT)+i)pf5c!+z3{{NGDawicZ7zp2OvJOGZ~4xfk(Zv-D50hO-us`{P-j2* zpKY~E?Z?a1P^_0f1@ZLa8~wWd9Dw*CR`0j$|CIVib`>K>HJ-fUyxbX3Ohs*Q1@ zbT*)J;_knCV_`qaMP}D*tn5S_jbfno*|qUZhBx>i*z!GgT|5gHkzz-_eq-`WZ|TZI z^r0kQ*_=02y;2zmva7L?J}neUDi%6wV_`6+ zF0?Tli%zS)tqyZxYgY+Xv9TM&wm@(zHB04TA=|<@?_;0`kC@kOyw3;b0WM^M|8V2z z#BA51(iD>t$1;18k&RqAcLk26p*vzza#E;Xk}jcBJMwq z4lfU33xYJ1Uc-fI*BTwSg^pxwa&=91U(!iK>k_KG)Z6_@Qn=mzC*&#rODCyC71guC z7SD$tua4Epg6>*yOD01s)601foN9V zoaTU7nyuk~l!irl&{}-r13iEFi?#YNe;sV?{&>+ru<>DKIu9#Gaudx4CO-?D=` z*>l^kbfj?RW71ijGwocptQI}tn)qC2mtwl{Y_hc*Lw#JVQ+}#>1v}R+Jam~}3u_E& z*ZGKaaEz$#_0g%sHEH}+0ANE7Tua_>Zl8_0i10x#a!MSPnv!_rW46g>hhR#&>+Ff5 z+QlCp;eAVjhngI9sp}^2#z~xRjc+6gp?)eV_-#2l3E=7wf3{cC)^(%!oHuPpeg`ju zz=$7gzwL}?)Zj(6;N`G(`3X)EOncrzz4t**j2EiUbsgzCK4Crno+F~(c31DEKUDzq zY1`}X$vd>fdgFcXz|ns^#PD+OJ5YV2ENOf}q{E>=T#;opj$C9B3rKhA#M0SnN2>26 zGP$my&&OKD!rReA2Qf5Iyd8@^eXZR;*>}eActdhjZRmeo7M@PT7+zz`Pu2e13#E>W zok~I}p%0x>be@j>%Cay^1rnB>Xzw_gqC??CJ5x=7l$GxNWCQU`7oB3Nl<47XK0^hq zaNSohte>mCJ;XW|Z%KQvdWPeG5D72zeqI3>_g(FJK3btJmn?;9d9QttQ);Br0g+&N zsK68!lj=u7`Mqa|RJMUoH8G`K-|G3;2C+*YdN?ss+ z_E}=S>_W&NCaxIQ?ehea!$rEl6R?qARKvY0mWSO;c)AnnsL{W(uW}%Ya@N?_xd_}R z5ds)I|3Abzr#dDU%`HG{t%ra6_aw<8uwef$;W`T=yszZ-Z@+ ze}dNcO}q86!u)YrL1wo-<_w%~NG-AWZAoOe`_N|j=mR}dTe@p)ST!G|-jvd#&xXe{eicK%%x@#AXD6{hlU5CpV&r4NvDT11xE_XQ zluuyfVxsBB_}>)OS!HA6V=q7|+Sp^`5{wQ-501|kVq%$_j=Kr*%*{%u@wm$-1|~e7 zKr&!f5}R}xM`a?FcGm9tFMTW1$dt!iMkW#W<^S0bZqPv4oX3K*oPn2qi_+1~^|$Y?$1iPOfF|9(u=+L! z(SpQk9jWhR;aHf|gLbf?#qRsnMBQgu=l)^4KgW{9S-eaZRsAbRi&`DG#ZEGR|J$AR zz+=JAqxF@uC`(nFGhNb+T;zGn>mLg#V6qonZ}b^-C?{uALaUH@soR#uklHT2W$z!f zmc->W>WsGr{LXQS804~}bjIzEg}qp=gUH#1iSF>h>PQr8UGb@q9ls#_K+eq;5b2x?6qqX%d=Id%pQtqDCdhTii%@gSABfB`l%~< z*C!&;E0MXe1s-UFqj@yP?Z(6@<~|aXmuYQ^{!4;3&XG4!UTw}##@d!dc|+@JDka<2 zm>+4_adq34hZNdR%%L3oVtn65H=_I)2W@+FuKl5Cc)7>zh-XDiWae<+wlkmcm0aRz zZ@ZF+rjjR^l#!KWWz7X?wGKP8G3BvNQJj zV%#m~;RDszCt&xK=Kd8s6dRN13q(x3{aTkmm4~aAgf<3acqDt5EUXnm#_G#r}{Mew;KDKhhsSVWTTd-0VN zo>G{;CblJ>=~CE7?|snG`Rs{+S45`|@@H+Es`q`E@X%In5v7WOH|gkh;`}(8lpCK( z1GrttJ%#hzZo8P+nhd}ANzlY@8lJLGD=4LHkYLkqpZTgccTn2n^CVssIbi%!^`t2C zOYJZFBEFZelk^m-tMp~!6akw`bM}=Zhii_ED=Z%*@^!+D%7fwog&Zz9luEJNVc#Uk zMCaSL*&2}FE;96f`!1%jpX>f@`~LEfgrP079~?z9xanHx?MMIlt{kEE{>6UE&L?-Z z*w4O?+bC)Ci%*=B8wO=l9HXe8CDLBC$vG5h@ly|ZwYMoAEL!qwHYKZYObVa)Lb^in ztsEY9-KN4Q;9EoU>-`6zJ`}OMVN+-&a97umQMdiZO@%S(RKoI`ynAUXm$li=`KyDY z-)@P)q+md7g9|-qgZx|kx>wn)n+mJ5c7)xwsqjA-bL{p6fbQ*%%M&avfC{T09M8mC z;KE)+ts#k+1=LF;RpQY6-AJu^MjGa$QX#X=h9^{7V4xBoGQ!~wUG2zC&cR(722e5iT*jmD4umvb_R?0hTWCFrYd}4%BI3hEkpTD5*CErX#uCl#V83Qeta@6 z*>rD8aVH{CmyrB=Mm|u!8QtPDeQo}Eh|Ss*SN!Z|G!mUSaNC@Oo2v~z8ouf!o4YBr zE%%lzHuD^`Qe6jUo&11v{0(-j!hII{nv`02_iYLuhv;U{D|vrDxkrH^vX?E&0pxv3 zHsky$b$Ww6kZ8lXj+`G>>%G7GFt=`f{QVRXbfZs{+NCQ|%+RnY2YPoWJK{O?B})_I z1ff%qv;tkh0HxfZwN}rLlqd?N(`?Uv>RL(-%uegbR>chOl`FcEx~X$#%X11-QABmv z${;%_FZXd9%OgPl5}Q}_@2e9o5)`bnHQD=_-CUf`UaXA;*H=V0BDd&3e}#OS4@GOw zf2OUt&(`^-8$ITCw!Z3&yM;kzn%|J0a7!G7Zpf5p8~vGfduyaZP&vJx9&JFCtmUTRvs zwkyyWT(!L4fL``Sb^5YB@9;JgaQH$JL&?;Kjitwj`upD(|IKz+K&p6-Jj>9pBRRRv@hvIu| zr#BUg+5dYypS=@ETKuMzRpjBkX5Acl7m9Q+{i}qO*VqU#oa0b z=i=m{%DZAgkPcZ9hmfH^rQVC+{#n0_+`!Sx$=}XxZU*!<}DLaAz zls$}GpZ2q3mXc-29JKZ?(xDium$Y;$DI%Iib`7dm+gjlCYu3LjcmYntqOev8)D3-L+p-DkHX;CGj#I)l73IHiUaGBDlxcww~gqZa>v zTXdai3kV;u+p}u`R6-zm-j!3z?sz=z*m^RNE2|;C$JyOfrA;03_}Rh5Vdl>NkN<3F zb_JtbkND5J4+~garO%NH*FQX=S4tp+?hzHb;A6zoAl!}2HaJJK3v5(^1k>DPWBd!h z3rNK%d(~J+(a4gq?4*tJ51agWje*o(j0j$=9{POIQ&8KhqM9PAp@nX`*x8a8$!FT{<9=mk(zZuypdK}`q9hkjfIH^PWl%CRbD&yRp2YEwIqhJ`>P%l_$MGU`eIS6 zh$g$o7RN8lGbse`gk&W-6bt#y|w$xyQeO&j$C)5cw43D>T-xgVx)#?DN^=wKO+Wu zR^Sf1(m$^4TxF}1SeGy)G_xjgP&V6@x7^y8XNfw8X5`EEP;|suy)HXytXYn$g{{v4 z8(Z3JLwxHOhOYeGwlSXR98|QS;-@z`u2yKF@nkOMh^n~S)%(_g&%EGEKT@u3)2sdnVw`@B!GZ6(H~7*QtA zm!J@igo24Ttj9ZP6y&?36EElHby4?vC!5Ma$AQUZ_)(mW4&=XsnDmczT*vQJkYH=@Bg)SBH*TD!`kcwvTi4DGew>8tg)hzMo1wqK9M9 z=EZo%btM~C;o3fl5#_1vClq|AQqtL{3E3Q7;h$Ah#+l9GPF-F8eTmci^9p7+nyZY1 zhpc|#^mXsNyY0&)3k1yl_LaXN@b8Nk*^kv^U22f5{L|xSC;T^jRR?QeJuVC|9c((g&_m+cFpGCJ-Be{?Apy`YX5Yn zUFXkPw%2j{xPG&EcDLH;o zrx(V>ruo|`Q*3&^g#{GEtd`B#96Cj6Impd$W^C{sF#xZ9L9G<@}vV}hLfu70i<0W=q(q~wxOXvV(b=2-pz)_9p zHao$wx5z(6?|>uOgSI&Ps%dF&s<8*+3%@Wp&|tZ0ec;J;Rv&Nu8h6)>MyC$UbuABC zQ_!34M`=>TZ(W%IXzAwgkdc!Q$=HsiNL*iyMZ^w`l&*F|(v5Tw$Y2|@J5GQ@Wr?J> z;tsS;)ymW_$HjZIgD~~p*|x>u?NpfV65CqAtMEpb;P!-`G|>C)`FIwqF0S5Q*j#uR>X9fdM10&+HB@R#9ki!ycg#=b zjw2kwr5;K-R`v$gxnH?2dvmr5Ho{vue^MwM3OTg7qy)MIEimED&HgdjkaXH z^*Py2Tx3VH)q`Z9IaVQ@(@hBHa+MRy5j!5>OQFosR#?mvd1}B}7s0T>`W>HWi(MK% zRn1yO+l%tsPAAS$A`sAO@AzDG1E9b><4ZXk(S0EM7pEn)#40=M>x1~(W#^)A_j@Xq zU3<^Ju0{HMKPM(};e7s3rKb;~lTJGP(Eo0eN$Mm2yQF*mIDsNS=C^jiNuiRH?P4J6 z3V~bH1r2`^P^I&b^8J09kd`={;^$$X`Cu^Ca4r>=h;e{BRhrf>982iwRJ!an`*QQy zY3@I{Y2d%auE?ohE{RLIVcV7VoLl3YoHBb`2farPOSt^j zr-FyrJo_%E$|0vKP*?Z&@k~zs1W6Mc`>OpA&s1myiwOZAC+x>;=VvvRw%6>ZiX!C? z=^jSzxBPP;u!n2&7oSiUX)ElB!dwv9L>-%}{0EoN64D1g;Z`dJ-@R?uJW*Jofu8A; z?OJb!c8pzqUB0+ys$Ks?VLF;P4Q_bC86A1J$j`^q!J=lc8=okw(X459lYh#M)M__B z5q2~fD|m((BbzR3JuG`Ac(K!lCs1C| z)kZu~mIjB^NX-v%7qimZ_k_lvt5H zo;H~;7vu{p$hOdX5ci<4_xl0{m*{osQR*R{c{&DJJ@(k5cqZH)1&;@~ zw--O*%)axtt{8Jgl*?ETp;>oryMl5kXj3u7!N*Q6j z_b{u5{70nBNi24EwAeZoSr%t4+NUl7jO2?(7a>Mo8Y9znfO+I=9}ODFtj$v%;+Wit>uFTlyC{ z>VnM!cIb%^(lUKjy&ZO>7HRSoJK~sFiY>6On)zO4PHY~OeKcksc@oa-E!Tvi?vD9P z{{zzsjx>Hq|F0A5M0TQ#5Zp#`{O(Vr)UnS_`4m*}T)9Y5&FO*Yx9wcg;B?MjdoMQusey^&e?NbAHK5h9 z^NCz(yX6G+fn%RAHkGq%un&D~N-&f%S@^2e*+)5FPZWU`ez$$>{ZjC-oZuI#$-t_Y z6Hii_#OjMqAk$^o`XncYY^nqw3J(7?J7==;eJ2m3edY@`8k+jN+CG&VSi#6){#D6{ zFJfRBID)>yLwBx@DxT}vVqw(@d<`#qTYm}e!F%{ zn7jT!28Shfo%bMZHp_H=Bu>CZ6-sdZ#f{Ng{G3uxC_Da54g+2f z-L4n<< zZwVfY98Rr>03-a1Vsuh+LOD{6j9$bENX3LdmW|4xNWNQ)hhw%B?(jE%|C^1i+Dog; z{lq>PmyiI1Oruz1Y`jAv)(cC>goKOjOd+f4mrjC7TjHXv$2kzaO`Y8p-|8+pW>dBl z{&n`_uD&Wnn2azNK)hfBmb?%=t_q38FaPS=qR06)uU& z4mKyIKtML`YwoE#cS|tXlQB%qiw4rbvc*ak^aYOJnq?Sj#7r(sYDLpbiL?7+8VWl) zZTCk9`V{?R0eX?+75{BpoH*o0le0TapFQ9JFdIsP)J6l5)nUj=iJR)LcqRr;cE?6X z7eS0Q)|4}tPk7AI00%A!2D;@k-67L9*LF(@)tV5)io9y=@k{koc3VdR2u;4s!uPQm zy7Cz(8BTv^j-v5hp2(g|+`2ruS7syBPs%i|h=tX$j#s!(gLL@;uZh-Jhl(_z+!t#D z5H!?S+8*)=a+Uwi)@_M9m17%?qZVO(bnSaz>>|OBVy$e0FW0^J-V_hJjWHDNxh0Q< za+7X~X9smgfl);#ZrQz&R?aZO&fS{8FWbUdaBaLV|isRgP{)+X>LTC0u&PX=^sD6bmdg7Uz zspP=C&vyH`*yKk~UM((i z8y(8;R-?aVN8(Mnp?GsS=C7~%M$f!tv6NVSb~NFqCbR;`_}7kYaV8(FI6kiauWl6% zF#T)C9n0j@Z^h&FMBw}Ybo`R_$4DN;THp$lp30G=pbm08ox}!1|4#nBY_^^8)rNmB z1#Jc@@HJ&OF}VKjY`iI4cB~+&8_+R#JmsbWw zcENkWd6hCAb}`p^MoD40%0BVaOEpeEP1FosWS{w0a%QHQ@#i^C9qFSGvA^(^pnSJ| z8Gj!@XZzJ|L+$O*N0>sAXVRk*&XV zh^YLn*x&CG8z{#W7p*P+y|+-B9&D~ZWFHt;lttV9^Ftowvq~UsCFl9SZkrKs4i z?+_@#hWk`GMqWm7G{QR`=s_AC`DChyIyV=ysVM5_JsAfwaO&C1+P%#B9c2CZ8$?v%W`r7Y;A%^FPIM5d=CXiy>>8yJRcVE zznpY$+q&GW@^pa0*FRYVf+cM04f(3RLE&k(F<>bE#7su&V;6Nc1*Q~O{t9oz!EVk8 z$YT7UZAoO}%c63i)XKK4)n|my(UpYTZMg&}587>e>}qvM7MhYrvX`TlRJ}M9OCXkL zxFh;eiXok6if-R|8DYh{gT>P>gYDRr2%C*`@O;iGY^#@Z489PZN@F5fUoqi5)g-(S zv)p%gS0r0f)kPvOcRG6&y~*Aui?A|hs`WZ*7fzmK2NF5|4%TbzkiX%wvBNn8@+YdD zq76Ob5Tgt@*lK;zK$}A;p%pW5G?#r^_(|Haoa6#hy&bQnLV;73N6FbvI9S7mTmO?` zv(*93&E|K+DPM<9Qe_~JpN<~(4eU|keDrlk?45Y#_FpM9_S>02C#NlEVM(}LuztV> ze!U*Bck<%bdYpYSIQ_^=#sPOOyB>fYfG`kV?LEiQXrg}KPiSPj1xL!}aNg&$0^jGP zAOd$VjzwGNZk?0(D3LP{z2?W&SaP;;@$5q4Q||NxQnc8`sv8w~7wuyv%SEh3Y4MvU?f@isU zYq^xV%MEqS*1Y|-=N0uqCNg?GZP#uMoe>G1$qs#8{+!y>zzK8xR#!qv%e2RC*m|Yy z3PBDMy3wD(3yH|3kbpnhTKFE0s7||i>)E^9KKb9WPTDO#$GHn_P{6bgZSg_7)qf52 zOi`V#+XBK;GlcUD-!i-1ThxHk_9nZdnvvqV2SqI%o3iyKVZ`I88WL!+<}d0!jK6Pf$EFmx3A(Fhz+d%+zo3 zf82QDn_dK!;`YeZG&i1!n3U{M32f(W4c>Jc`fJ6mEr>>Lg5aJ*=3eOVQ6sp)?yG>& zK_V<2u=}?Lzl{#_M*KpFWO>A72bf|w3MRawk{1`0 zCfZ%iQLBTvgaGAy-QEhwk`<-6RF%Yr)?4-K+Iar81Aa{Wao#LDlwB5|ZJqy;jFg1^ zup^3y$vQi-HREVKF`ITeE>nGpD?He0pMD5SCf{GrWfvGzrEwsGSKs<$@Azw9GD4eUe`l?8+{AF}5wIROMbP zNk$1ZPII7ql6^ghUDiHLcsbKNPF$)sad@94Z0H+gGLqQxd9>DH@JiVveWmpQ`yx?s zH}v4l=$AQ=GWgk7-W~RmHr3`So;_b*247l6nCCvbl+Q{evKR^6pmVv#{Oc(Dwt`Ey zr+DpG?7Mgt>~*Qe^L;|pDYl`pnQiui_k*7)HuxW%@Mn_@{-^ldFDdCSU$vioOE!o0 z<(Hfc*FlHbQ-$@xeH+f~s;A;W54#nYHq6meg+m$W8FHsx^HlKac$c;m>x5+bYoBsn zM}gO3VRbxR_f+sov*GVr?E0sYIXZ@t{)VUG)|%h$SpaW*%6Ts7AT#fUd#~ISA1eCd zwA~yd!HpG{r^Bq|E%8jcfl`r_XM>(9yxsgRlt)jM3;8nRc&FVKUmWn@Ccf*p+vAyh zIbLB7DCF&qfJ$p2Jpi>%d80Uz!8vB^4V7{j63=wn^pzcO$c83D;g0JXh`dk&h+)yK zlr$_mB8@B@vJnh-uu0xzBfPhP9yygYbF6dQjC5RJ4Mc-rqn-+`2D^uLW~t^a-9C`Y z4|>c~sYOG}iU>b8aa{PHA^~fyuzQSqD)_WcJlqLJZL{&YdXtw(W|$D4I#mfBsNsr> z-9#r20<;g>B!|8ms=6y+LpRNuXysic>498;12knRxtm)W$KT1S_ZQ`4O&iQ(hM znc>}%F~nwGCO^36E>a_iKxaKwSo$IVht2V|7L$^>Oy|a`kZz58PDk#`HZO^h^WdZ{ zsMb>Xq_9$Ol`PEp6SD-PabLBxC5i=t*>Cp;PK1qDK)){nZppT&LbQTKeA@23wmAB! z@Tl;vs&z4w;y9E3WVP8Ey_k;FlaOS60#?&q&pI{c&lPNDif`q_&LFqiQvXFvJ$5*2 ziFa7cIXYW?-Ei6&ZS7CF>av&X&GUx5!eOK!WQVh;jjY4hggo~IV|4{+zJnAWbl8^X znvw;#IOwd5nYkKOm!}ks@G-VJ!JgN)3aMa?1A{k4o2^Z%RSeRq7JDdrB}H-(&(@2#3dX2W#&dMIBg{kuAba#MJjNyM*7qvGLXYx>`q*l{MNQN z$7)z96B9MKa?dENyG|8-?Lf{5h6@85Z~fva%?$iM*LL8TF7ut5nCY1zUVpqHKnCK8qdNDO=>+mmQ*072sZe59tg|e=1(Lj^&|$d{^}GD)~Qo!BH=W=XM~yc0u`>P|aLYI-}9LktfQE1)mh;Y#pBzDxj^LH=x(WRwU6Tb zLm>Y+J656Kl5X}w;$lh*ZfzI6>F;6hK8db-fn9zfN_q0BzwZ2x7W>Spf@`H)>ZQ;9 z2<={KUu1{HU+K#nu(MsN8d`o8n33sTMp4Q^aTO>V{n!3w;g9#*rTBG#IG)$-n}9}# zB?%#lf6BM<4E-I~i5zVY*msGi`BUwC|C_6>&3>p5<<(tY=HUF%VeW$I@s_o#5I=wR zwWw)b(v1gcr~Tq{9y5BZ*|xBR>1eU5wmH)x*Ha7&#kJ(>cqVhW3{9xZ?V4>Vm6>stXjtjcoxGgUC zy0$jCdfv3n87SXaekZxx!_C=JKIkaeZ`l?K3M@I1A#2dK!YwJ$->r!igbNAN%F%S& zwqOqkc}Jz!T4T3Yt)ylx<;Cod#O}RKTug&~##kw^xkc+t?FaW@L;cx651eLA2Z@(B zaECc$nDDA1j>BVGNE*1Xx#yz?!;GjpEj|rp`9@Y?5pKb$tBBT7+k#{5YN%^!Et!R( ziZRizj!-$Gp&8^~Wv(2X)2M4>;l@ReI>9}g6m=@Rg=io5y9v?Le(c62Vpq~jv2joI zc0`X+?!qQTPn?UKjFLs)w!5Myq#e$|DOC%(_!n!iP1}|dYHAEiii_zv(Q@D6QNu)M zRH!OabTN@>d%k8fw*|Ld(y4Id&+^5o=?3X@;xj2+2=oVMaKHN7<|g4_Rc&ruW%FXt zl7yhJtw(G@wrQ^^x#|}tsmKzG#Shjg<|wb5gLZ%RAsYbWs07%eYAs}i5r6nHYaz?% z;@m}~i-L|GsAgS?W5_ILwcCpLvaE9nw^DtQHc3a|m^FN3&K=u^Dt%g0wqe&`R@LRQ zH2Rfbs6^&{=-OIxV#@!@mDn0>oc9%i$ZD9&M@rZO*0HT{q1~&k%h8KIpv#s!d;>k& zdMmG(sgz1O%ls@{?Oibq^g_0#!jb|pi@ws( zez8`zE&CuhcBgIkU(KzBb=lz~O}W>0W^YJHuX1d>uG|We!E9H&QQt1hkWjB=8G8O# zhZ@fxu(@tK$>Q8n7>g|F*qVH*b1A3%JJKm;!!3C>suVi*h2B zyA=lUPzBZ-gaOFz2-xLTIP7bTh8Hzb^dqDEQR_hRWK4tdayB}I~= zS`~Qhc#_(DkGDG!-QlOoj_D7;C4DON@#mMpv-u{Kz_w)D)^n#ai1lbok z&SWK*7_)|T_g$?X&22x$XMr{B^t(z-IxRkc353N` zu=_rn?lU9d@r*lai_OLw0j3kkbOc-{$4|VhL@v z1<}G&@hbg1c@lF1B3&>>(LK8_*J&!XwRD@_?}RZHW$&WgJoBjC)MbkuoYB7@Ki(e5 zS2gubFsHTu1s)1sT79mmvMkc3F{j9p!*4=Y@U?47phy4yccW}+1)zMU%EDSpqG?Lo z3Tv&#@dO~s#n2wl+gy1*NJ@pAId31USiYMRf|TR2V3t$u$-N?A-2 z7H-T@;M^sm5G`e!5dEr*DnVb%?wIkO#RPw|SGP-C!Je zw)@(W3(I!ojx9V0I&ItOv-bR3stC6$km#ILDI^9Gu|eDOx%v&uYH9dFj9hg09Aa3$ ztmo-U?e0i1!A!RfG5{`b49(E@kbO#_Mvd>N^#l6r$;7-!uN@Y-K|p@a4e@X=igl4sc~_ z!Z*gwCFHY|LFK*t#Zhz6-jDWPdrh7&hwXd-0*w_}QLg9ZTCfkQ-o?AhKJc1-n3y6$ zP`Xe86f+P6OZj2D=+9&X;Ba+y<0lnx(owiG->30R zC$+@XQ(T#!#WOhqc~+W3_Ia+Zl&Q@GjDFz=YwK!TX$g7Eco`qR%lBw~R927bdG@`(ALv=M z#(t=JzI>Cr5&p*{-g}lUv!9Z9$(Y;SZa@1Brco~wh<~XNAp50nB-qIbVCuifq(jI33!F)vj@_Y}m} z2=S?V*RFofvYt z{gpCCz}wIJo3|InWF8y+mhHj3)exN3WrJdzqCf{m>YLlGi5*nIT%8<7w{3T`9h43^ z-d@2=r8UNRg`IuJ_QDVh{`+5t+29=F-i{@lfkUgtQQbwu1vx?0fw~^VK>{rZHIaEgZ=t|Pm4d`R4Ch|ZjsnTrh z_QKG@pv%XV0;JJKM-_Z^v1pC2p2_1vTfEOE_)Hq8MzX{v<{HcN02U4x+pT;6Cpj(# zdM5qvc9(yGw{^{fHYH&}X&P1)o3=glaLRbh(aRD|4{RenVNaP6AITOjU5PG&nK8D9 zu{cJY@@D1O^P6o>;_b2)zEIKun_In+#aT8$ao9>OGq0Mqte54gw*}h^<2YloE%cRY zt*x>9a$*Zv;``$(&lv2~x+tD0%{w#+wNjcQDdP_?fUWjGfWa{TQ25DF7~*0ov05MK z4|gxHdjFHy4r|w$v9DGuJXQ40T#d{w@htQQahrux;}E)!uCn zOq2Uh3{Hg=%&`+&HD=vjIpyan(UbT*MrQ}ePqI6}nPI3~Zu69T%6i3na z_%e)ZSGcpRb)>r6-9t$k^P9l^{N9FX!JFH2kR-L}hXe1lJhbWNQ^=55UB zV*^pe#Z58j6={;#Fp6N_?2A%^z^Wa(#k-jgj$s7028QG&^|o!RI&H&Y4c;eDiq0Ap zT3}b~@YVVw`Qdgt+1&f@w#%obgr0Z?Ra-o?a~kvoZ&5?#`Wox0@K=~Fw%&`jJD$m+ z^eTXpCC~OcG!!#+$a|xWD`+I^9C`ni5@Wa($@-0G%#kGAT8+5h7 z?NlSn(&KA?G?3|xi1v?rjr7^Eq>A7{o!t@*9gluAg!1vdg?Ge>3V*Wq34i^LZ=nO|%gS$?T;ZOKe($sS z<6Qe8`!7sHadNjG6C?0>(*3De!B15?Z5mg8{G88%fmNN+U*ee#EM?<5$@Tg)vkS}FdToJwZN`>CR8NYu*n7eOz4t8 zBdcULJmZ{_h)O=W`yKn@cCsa^vo_LW-1JQFY~-M+phRrj_k6o8X9Lftuw&WiZ-1t+Ay+sH-Fc_o5e=2{R8{FI zXK#a_DQwR`&)`Kiq?(HMqPrLCXuuUZ-UuYjSC@S%Y*_S(&7qi$$joHJ^BH?et8of3 zWkmF#L+v%5VV#I8Y-F^Sz#*;FG0f|u;#pxifkc_5##HT;@I&^&12#5$Ej|2 zgIr;FIZmm-QtYL;s5fm|K670+7Na+9dQ!u3^1z|)pW%pq_y2@vndyr;ni^@d;#19n zlL({29-EV&5xTEEDr`5$Dgf1<^rK8Eea6kz1IwT`c={pTj@UQ_Hn_ ze|ApE-wssBiyTQ(|H6DOPLvfR{R0(&9Ac@8e_Wh2wbhJIz}b7`P<{5SveS@kV?qJm z2oh*|rgAW^YiLIsgwwS5Qj&oxJ*IpDTe4@}nBZ5YiB^Y-u_07zPjn#ep{XHQEIui$ z4<{Na4y`NaJoVl`+VY$g1)Nsc%4b5ciy5sOSQ%E}t`4N`MdVu(3p8eot<6~>c`YS{ z!9x`Z%4NrgRPf6oTbHY{=uTUoU(s_1Jj8|o5!u6IWm%<-33fR{bP4XUP1(oqerKDj zZ(N_Oj5%ALDf#|n1R$koc#=UPTXTj@I)}H#m)gF5Joei5Tnz=Kt+E{nAl7xM`FAE# znI>NG4Yn(LESZU)uPEWL;)>af>FWGKeuxUGR8H=xz?S~T*^k}T=U%i!qLgbM_vSFu zWPRw3ZzDhx#2mK+ey%s7^6|fxFSEnGK5b;wIpXsoQ+OMtU#u^BC`NR)gF@Pl=G5=S zv%B4nRS=W5Ujf6%6RE`k9z9?K&O{u{Cmc2i{}^!eJ4Mx@SHRc#nbg?30H$kGK%dk_ zhAZ9a0OKH6lVYxss_mTwqpo9BdzYQ@CbHX6#!PXQoi>oznA^e{o%LBZQU>>2)gLS{ zBT4dkFTj&c{)D}spBC4+^Lf*VPB?&wG=lm5AbKet1HOv(Va!CAz9PGeBO0`*`cHT? z+eZ$>GCa*b&K(SmENQK2DcR1Equ`Y3!4%M8P!p5RcwdgkoJ|E8OYeVy2& z7;h8SWj~oNRoHWXIre~kq7S&?bje8Jtb)uxMTg>iq}IVV(SA@Du(U#?iII_@pV z*><>2WFL8YUb3rp1Yc7FD}p2I)jOO=V$0x6;4XOA>~JdAHp23w3@lKMa$<;Cc-5}m z5j<3P$$EEPbm4)fTG(TD{f@$wjr=#eA%@@ok99a5-{`*vdKSpGdDD*I*f^YEHXH2b z9l@r|sIgmel%}>;q8tXrC?1O-H?5cK)_7L9QeA?e`L^nra(%ftfBTNOgKF{0#n}R^ z+~FgQ9)osoaDs79POcT`;*gDT(C_)fj0H9} z`Xsm!<%L_H$K?`}%0@iy_~^+mEk6WYdoiKr}wob;;>_CcZ$ z=NaYLzsr}i1;L-oWXg`hOgD3IO-mTMs4sx|Xu7{zK;+{L2Oq|`%Vq`uLUEovvywz8 z>q!|5ZxQBiTyBZboEu+3QbnLqQr>w9c8!9&ifM0w4+O{2U<-EyQ_hU)sRV>b<{v7*4!B z-b=3afT@1#sE~v@eK}_>p=RE!xRJ;ON%0ZOt507kg!%?I+sZ^21(R%bjPoSI=bGH6 zvx=Q?tz+*ue;;BGRS+t7g_x3lTUSxFPB=-*B*ZSNtM&25;brHE&EkN6)i&e^T{SDY zM>gheReg>INMut$)5E#5If;k30E}b-flg8}l`p|+ZS@vtA0C27*yi6Bh0=BheKtnG z9o53ad-}%UUr%Uv3PEXSH8tWDc_SU&t{j@Yi7obg3`1ou27Mt3fvBb>4Z?d*wb;tv zm3biKMYh{9{f7qS0@<6LHZ}9Bx0(;`u|Z=8;+fYwFCqPQ$e*EpYr<6GTDrsD?bN1b zJR^<-bPx}>SznSxCwgY*fsQ&ZY96eyW7Wc_Zuu3F>UP`@`8o6KM1ncH1_`)7`@@{y z?27qQ)uVR2buiJ8Vle=h+mk$PUK6hr&v0^O`TdYO^D9hAX(I#Y5d^(~gbe9Nn$pXa zme1yJQE`$(<6L$FYle;i9j>w~sfg`+`KCyh$y#t7+WWafXTtp9&wSpeDV-+F_Xm!n z(PPHihe?wp1SsXpKB@=^!wD2lF*N{`Db6EjKCULOL>Fbcb|EJ(lN=1C$`f65`W)!_ z{doH%kvt#5w@>rUOcX`cm-6^&T<=aG_9A?qteOIBmGJvuG zC<^TR7_DeRxmhS>YClvE6txo&?6)6xgwdrzK{$2p!0}Tw^tdF~i~bzXgu`QCU&7=s zj=g$zC)8li7N$tCJZ#3Ro;_O<=)~Gz!RlQ7Y?|9t%vDCJUc2Vm;96>0y&n9v(Wvlw zLO+byb=9*t{IHN*AI~yDin2o7@ND6D(ABN78=no{Y<6=!F2;6KwD(k_U$i= z6=CqR!SXGja&DIm$xiQ9Nw@|Z8lUL!R%)^%?B!w4hOO_4_$_hlN*Q;Eqj&hTg+*L2 z%|>|V#ZtIun2n4N-QUmkQ^ohE?4Va{R1QwP%Wi73jLE?z2++C8VvUVwMNL))JYeH| znBr5-9L7fzZA&SOr9qjHn4I6ba@jVGV=m+SH$Q+jxw_9*nZU(O6fP1;%%cAOys}ki}B$Y0{(9w6f_O&NqiXVt)uDdRPS#5$ud?wZJ>T^mY!)2f1 zI27OP#Eq2 z>qu6$$~NWP&?d4^%G@?PJt-sc7AMfvO|~_<51ke#@-YVCPPT2{BhKtajj+9fRAIBy z(vwQaySaP1?R1zsJb&`8m;s7Bq@3sDOYH$kK5}?^;n_+i!J~DTC~= zPvb9x?TB}Xya7FSqLN_1z8C~ovJzw>E7{Q`RF839Wyc&UOi@%3^VzGs1%|m3{^s2+ z?DBs9jLEbR5Q)z2)UU8#L|9M@V`8UsJ-a(m= z&ve(z3G^sh>2v-uCCcr+oW0~bDYfu}&iP5$&>@Td=X{RN*!7fd_a?*zlrlYqN%%0E zkbw(F@sE6(co1~j$5oGqw30C1`|U!I1R^yX;bQ!fXIYX~}mCIV}$ACpdDvz3z zQret3$^HFPc7YieUFOe$JneqvP9krW{qn1cXz!^Qsp2JVXL2Z>+(|l+tKyk-5pwxZ zIm@ee#>vAeNKRx}w`+E~V(0+3(H^^Yr}IEX;)HC-Wm4|jH|)BdAt?`8^f$YHXYeSH zc3XR=-4LJYf>uc-;)M8E0mnD)EG$DugCC?f1x(0d#pnMx%X9P2!U8ll+burETy#AR zHmI7?9+X|)+pX2JeXzKy!`f%J<&?rtm&}SggEhk)kdvsx?#L!Jt*zK9;rs^2thHI* zvLOy6d>~4JdK>Bhwbj_Lot4?*Kf0TXrC_*!S=&aGC$3>5Mrd3L48=n&=Bv)n@ zo8X|dmdfciu>zI?ZtM`%hMMG4*CB49RZHy=XIc7GdL8=E^&5R`d~J7{y^~PSV`y(W7i`{%CJ# zv3Uvhlv3AtK~9|D+G4L>nDdaOnsFso-j|KYd$UXx3h&PaK)>xi8H;v0w<=Y^VXomy z+ZoK-;tKW3)1(kW`8X-f&I5^pVnpZk;liuUT|&_!u1~A4+Q=k}vog19qpuM569p4C z?K~?-brpNHrT#a**5j-tzH}~Cer+eLHKzmluy_7`Z_277;&#JSV@_?Ii zx^uDkZjQ#jmprxF7H`Z6fU8gpQy5;+XYuKhiE&%?rDrb^1F}7VD&7-Usy}w@EV982 zw(rjBL%)G=hPGXPmi%G9J?|^s?J4A6*co=n63`=)N4ZUVVw_;FL!}`U@Ah#<|8|V+ z&6S+;XX~v1E0&Z{L5nvzkl2Ekle3LnCk`detU&LobZUq5tWrpk9m!uaR8OEU7FE)d z^k$0C^nI+8=a3!E0m`7|M7LuV3f#F?W|HIC>xw35@8O4q>nw@ zK8>#B6DSRjQjLF>?Yrv=sHk8@Md>ZOqCK2agcrSYTnk3qYk{QYBJ}jNd2l14^k>m^J&M z8beuEeoPdX+z>*>Px%d|R(TVl@3Eg_hNxZc)9=)RO7grmCVxo{4ejj`LV^66I+Lt&6ZfZG)U;kV=&IfvaH{Ncj z-m82rIpG&QY{2)YMP=3*%$eHG4yXCpUe-UPMlK-Eu zK>^Bs0`oA4<*-B6*lpD`M7Q#&IAFKu?CYASeQ0-N6XIr0=%|&a&~wfQ`kFZhSy*Zt z5<`(nk=V?k33W?btqn``bYRrB;nA%dOiD*KqU!b~)DfiwPu#2{6XthcvQdGS9&Vd4 z(Thk#k&9v=#`>TQC7s^5Y8CdY}(?svNyPe3$N&YuE)nI#a z8q;_&C4ejnPf>&PwMADmEhk7xmj;_2pkeLi?kNA-jCdx_NkPy$|7JcHOr*+Sxd=Wh z)>r3i;7A-xtkvzX~ zlh&xdT47l^13fan@mW)>#WLiFr2!SLTT+iHRMc8>LVV5X3Wv~dt^Um{ zg0!u@daoD=@Bw9XJQv~(<(kQtffcLEfh*<@TOJct?ZFpqWxV1Xqh^?6a{hvOwkA6+ zUaHcf9kI3ftY~pykSX zvGN|WP0wYtDK}p!K2n&0&51GPF=<|ATPh$#HT${Sj@Z_~w>pX(TCA#gu5C%IxC(XH z_UhZcMQMe8$9DMA{O;i#2Y?NvMEsBLSu!_~am zWaU0S3OK5mawOlt_b8ZJSBtyh_Br*Ct+&OFCZ$a2CQ?ZE|FQT^7C!G$8S&*=d;GZ~ zuoSh2=Z|A~>_l#iMtoA5tUu2hR~b=FEa%lZ`y*&nD<-C*6E|F20d%lN5{Kk_h+QsFS(y>WdcheTN8%;`eqw zpPn1`9*tY1Fmo%S8!tEF59oyE>zUws=SL;8~J<{ zwZDCmu(GYCBv7RIr~V9okW6>7%07#B&Y3I4$IJG)PoSZd#HC+kKkUz=B6%DUzD!E| zg9^rf6)lRd87&mp|8?Nq?={@+irr0kzZ9)y5R=!9aQIF2OahIM@ommgRRsu1`p(gU zGeM!E@ADIR1wubW!PpND7WW;?Y(FNDWcsPMp8_DIVv~S;9Le0d)}MW{XuFTwFF9zb z>x1EHmn&`LQTU==6~AzjMex3QS7ADu>k$dB*;S6ByBFHEyRN*@rByA4ESQ?>{A(OU zyQx)u{jSjLv?~({nw?#RQ|crQi7aL}?kX3~LP+MOU2)dVNAJwJM}>@C!NQFG?I^p& zTaNziI2*Lf*(+`vcBF@M-Wty&nxhQXzjoXIMcw;{cwKM%{)>o+h=_=Yi0FDfUGwYq zsJY$db^Fbq+dO6-b-Ozrx5xSEjEysM=ZC4g!t3rvH$uD-T`99^9LI4S$8j=$WfnF` zL_|bHL_|bHL_|bHM8x;;e66+Kqvw0?<@@hT0g6H6mbW4 zuOeXFme1bWRlK|uaesUDE$+r(RSbGUj@?wV#U{pNtJ{jsV%B=p?#N~}%wSUVa~QW7 zPnSzFL&sQCr%mw*+cNf9VOeS3*@EyYL6IInmxFgHm?M=Ek$)>JF|HaW^lfsCK%UQ+_(Q)jiLJ z(6fbfO_9 zg@avNau(W3{}9&lds}rmsi6vy5uZ>bTN7`J*g=;HTyn{3t$*LGycQ1xdL)jVvvp2@ zSounX&-%m#I95tI-E=o3B|g;L)g+&J+vp!r<5w19tBsEehb`ltw0fUPH~PNDcwNkG zyjGN7Q+~qfqw->>5GOb1*qpVfxsc5cSW6NV=_;+YYCuc^_~puK?Nvvz%0twMF&(+d zN^T=mGl3V0x8)+qiReeq)=S<==N6^Y0-;_(1Nd`Y1KqGZ2^KjBLS?`XpW`(%Dw z6apf$bM{m`a~C+L-QE6-^0r-Ds8<2+pzNu`K$O^adpg&%9*t}9d4DF+P`E0)rtnH7 zf_OIGXf#E=sv41Gp>wu3zmXD?)l%HPb_=yEa)*?lF_`eaB$u|jI%4tm$CtX4A7>$? zy7Vy1p2VCCMv!W6%&wm{)Llhmb;A1MYd>PNjK%EEq4geLU3A@SwesoDrOZeJ|?5M$`7q}#2hF_8o}mNSQG=sGUHs*BZJilUB{ZtO3;JNW7!?5kWl z?sCYhU;8U*W$c@nqn{AY1$2H6*|z~RQ7LaKGUdCRWoKJmvwdGdB>I&te#Cyr7Vv}Z z_G7ltJ769_HS$vhGx67aHN-jnbIuiqAQWi!i$k|Xdc0r#-_4uB!Ee>x+t$>KUzpkR zMd0vsARzt<|4q_KQXKHXmCuK&hV_J%U-i7p5;CXA)GP0qtDi4I2}y12ns_aBr`*fM z2(NwqV)5%KiZGT^s%AgVuFJ7HxsxL7T>pF#tBCbixc?2$$3aR#uTuHw#^>W6u<(-j z3%e0I`n7|J=Sk`jA}`$=1nP*Jw2O|s6S zL^A$v5YLhvyq0h$WV)pe8u3nYAy}4;J5`F)IKeJzXt#9^5tj#CL2P}6)Z(^Y3PQRd zupyADI=iE`(dQ*DQ+%OoeGqtYg90r|?u+__t)&o6W59u19S4&B9@9i6q|z@UAC)-gi!(^gDa% zaw7Fe#4*xI*>2yC+qc`E{P<6>s;B+qRr9Ii`%I36epUul<@W;g~=mh4&?}onEG%6MBL+9`cWpzkqQNh zo|bdit2UbL5PnVu8$ID@BXSQ9o1`bn(U_K=56@7pqu6(G>0wpRCN{QU`PZ&LSK(ER zD>bmlcy_)5LE33o(Z7)(Ks%?wzp5l%s3zOp(r9Bovjr_}xW8P?E)Y=iZu54aTDFXS zdnXSO8UE0ix?1+#m{mItpzme(guxTGH9kkh*nbevE1${%_m%X&_F)XII7!K(uj8-( zQT2@EIxONjLTf+H8KDDkiFnjL$<_G_k!PP)A1Ui3B!`mVGbgV{)>F?k@uzA7EI*l& zLbm)OdoCGdTps!|_iPg;#bTQO%BQ!$FOFYVgSihqjt}-tj3CLc#7z|&pYEUECNdgZ zq;~x-LDAhFIvP*k{e4ad89p^@!{p5Re>2muy=|$_KyllV3 z*8_MY%avddY3JAMUS(T;i}jTX=bYI-mo~&f<%UF&h1erk>?>kQGl%Du`&^3g+kx-M ztKymGoGdm^yE;0P;#ATO#fl3LIBM5;Xa8{LRJ(RxsB9=z)`t1-x)_IzcKtXotdh7# zB+0iM_7&lVswt`>!#=rjU#gTS5okC0FBH16uid;aHWo6N96riA-V*Q`fmT(aQth4!c@AJ>?q|LwsXI}^`By*h@Uq|4TSvVXUw0fo(!=w07Q#ij(d&Pb=moCg^96`Qs%q%ZOn*3@$m z?6Mh-%EsnxHq-w({jR%gR?J+*(ND3{PC~P@_l20crDhXG(m7QVpOM1!=T`V=C~ijc za={@;$*SiRN)@n*-2-d+>ld#D@g{{JPvnVi#TH#YKMWz6>2Ars2pF!391}7EkxqE2 zkF0PX1PIJH$YskM7bK*?;%&a25kwvfaV;cI;bk4V@xu^YUF9QM8BRojUw$S;eUx<>9zpiEnAPY-^&Wt~l9lOTf_Qa7Mw{ zUbP**w^|x7KH9_4Ae1rq=%2AiD>@kAV7t^QVrR4#)yo-F=D1xsVoS+^{aAvCk)c^C z8Cb&O(V;%m-Oumo$05@vE*V}@iDcm?bNYn<<=?3Al@naRp7NpE(dOuklPdnjeyJ7str;35ycju<%--%8gsWTQ!Z z;n02}2S(%+DYII3vU=v`7iIf8oy*?d-GYdF))A}ACjK_QF!=7SvGcick+nM-NPY4~ z0Gs?rMl$V~!*(G*yKj+=W%mf^9h>Z85;?krW_vqlfSKx;y|XXwd39tRCHVs$$i85I zT50cjm$DSw`!Neq!mH4lEOH-Q28{d&&Vo6GNPn1k;R-F?!$=RzltpK=B z52c&RAR1r@c-y&c76bOiFu&-hY za``JFCsgrucDA`~yM2=@C4~i)puDo*#xv#fJHxZ%cy@f39dUh=5<{5)zE7~X7oVNapl** z33?8Epni*Az0{50W%hy#KVs+P$ANEUYT_$iDB=a0;s)+*SH9r#PtnwZJOvzG^+I91 zwe5EG3vrfiBGR|ZuBjTIWt$WpX4mF3$2ce&uQigq-t298#!0;cseciK==b)KHBC@#zXMDN)&sIA)V$ z0>unguyYQcCZSwj4wQTM37{FBX;WhYzK2166{s^UhsU3r(1jT={#i%Wwoar;j!kpjMw<)PktwX#L)p322$?`TtzQ!cIcCM+o z$2J8VEFH{&`vn5(=G+;$(H*ju!0@FaHd=fH&er(WU06t_#97u}JtK`TSXaWPjvSLJ zU!7ZmR7_h|HqtzCH zpJ}7*%qBJUn~7B9Y}plKqsd{uxLvS*k2!qH5lnSl+2euuxG1omx<~7p;XC>XU&k8k z9rom9a7&1gW$bkoH4V7`@O#@Gjfx~u$WyWjVam3r0!N~iOBzr6x=I^{zfLsJbwxRQ zspV(S=E5&!-|wveAcr?1qSxb7Le%FH%^aAD4_CC)z9bWB+dvKbtKN^7d>#^pdvbr0 z+q>82FatY&U$EtF>#tx|sAQ4YZQz9}CDE2G1Yo2e^I)`<*-vGQ*$y^TeXkIDOk7Cl z!xbQYr;sc*l8m!tY3#yX+(rZQ()pc;Pq^$CyO9pGlTIL8vG<^BIvp((+@Vl81;d>6 z9#q%S&Q*gt-%(uB`Ae26bq;<7eIwo!Pj@I=7Dj(_A^RzwnNSUTW6Z}P;ZD6>bfi<0 zs`$yh?aS9xQgy%MlcWH0RW|6|+r?`XDJ^aa_&q zBqoebX1+I9`?5@+w6-9 z8*&Gi%yky(l-2IbT(%|LC()nQwb@rmu#+cGvajP589uHPRlv!<3TI*1W=L-d9V2G{r-qc_elU1*^3Gj>z-=E6d5WaUM_ zIRPWbJzd?hzeq7NW=*qO_ZPW=6mS^FZ;K9`o%&Jl8|QZaVm|nv5OC;}8McZ3qNaPB z-LXH!9@5^I7|BTiqaTH`boSfic&4nq9*=Xvro=P1^y^?|Bqon}R?W$W@8-VVA-o@PDQ$1_>ExZiM|5bQCpj?zSfX=jwl6ZkrP|%ZZ$536i4% zRBMbLw9d|X+1e{8dr?igR=uN|Uk9vCUVL3aLE<7MNQP6@8T3{sibtkSoM_u}N1?7( zN`>*t?>nN^Bab{{4?959XU?=oD~g(CJNK9C(O*{CuAnUjE3&spQ`;emQ`uvQBJ`}d zGIZJFF^9gxC^{kk0PgZk88iev+XZJr9&om>+sU*9$-SJFrrs9>SkX`pA zFTiCNE!)#o6GdJN9>v$@rj7#!xLsBBkx?5$Af1$R}v*tx}X{yl&lw0#w* za(6DWy6ul=S~lImrE}?VuojY2Nuf-=x#IVso9)X!Ai4AO(Fd9LSI<1ry_m~DK7%Qq z2E&T&9L#4$yQh%mQ5%Y9kyTh*%!Zu|mp8O-!mK;uztI0FZN_M{mm#~T^IQX!pZweGZi8^I|CA2C&P?EQE_I-9J z555vG{zD+pmDy6i%QwuurL^OJ+#f*(_z;t1oj|&E9y!k&rJ57ddd04MF~p+c zmwwfY>9!=?1N*}r7r4{$o& z@?z*sR^bhDYj(Q&A%?ikf5{1o98$MOht6JoH|tm&vn0W3t2XJyqE~uz50>*iaFfaZ3zQcjz2Icnro=L(+wf`B_fvD+dt0%~OpCGPiXbDg z{6Cx<&v-GEb2z-SMl=00(0mj&IxAl;Y{KYgvtKNd;Um9WZ*%-jaiJ#P|6E5DY2?rg z%<~^ylG^I*o`eIsCYfM`C%2}K7oS|@eQ!bCxg@EM05%-}1NiSQeR14mNlXn949G;w zvLSkF8Y1W&wdK_WPT^Xnk}|6K9HtE!3sF`*s2%HLEO}0YXUegU#rN_owT7^ zrLB$cMD0||BsUG==dmchCw&nmot_Lvvo3){`vN<`LeOXH6P(rUiq7BQ>x>xNAQvXv z7_eiSX3$!1++2$N5{Xq&F;gjKzs5vuXC0cTrW~)_;ar~HoGmEfP;V_UKM`3(5QAMA z$y&W#TVtc-i}w5q54;xZsOC=bM%qim)m1%{4`$Ih+1A`EolPCKEgA^XuNI?EukFam z-wQ1@*~1BMtTHzjd(;u)6^>yIJ9E2m!P-@gsXWMjkA2MFkM$s;Kknp z@!-h>h1&qeXuNBV*i-QguM4z#<*)62F)k=X#gbvO-u6TzF$Ur&naO#3+Nl&Hl|AEQ z--oM;JsVA3Z&|3o_9j~*Mo4wqQDZ&lU-Q8B#TU}lk#?gTx z$+$CIfpZ>vsrd1*ja1Jh&-evvwA$awVxjGlv6d&Ib&2NX^kHk;$t0PsCK5E~T2iv( zY?8`?`|n@9#?CoeM0PqVjV6S}D^y57Xm7;Go(9kLX9tl5iNLB_E5D*R^ido0!Af0< zi#gaPvYL`o&|ej{`A&Q)W_y+?p5*TNZekp*hX?t;H{J@i>Zj!UPEBoFYdTc~xWhhh zqAcnRg0FIm)wglU`6xOF^NCc1_HhNZdpR5eU!Nq3FsXf-VBi_)=>R`V0>;fAF{RT! zcXFG4*WLC-g&mG)KzzL@vt@ewGS^@^C2qd*E=njAidSN;63!(u=zWu?eP$)XCL<>vjpLyV<^RDo4r^lv{_^HX{vcP{!Vo)yxD@^nZ3Iah~kFu9_C z@lHzCiB9{qIw(gtC`kuck)O2n<+{SEokv=-o;208)k{LSqUWMAb6{QTv@8AJIB|5@ zRXriSENp9Sw5xku{*eGFR}(J(c1_R!TmJFu8renI7T3l%VYDfMGS^l8a0dq>2{+fr zGvyP1JvPJ*J>`IHs@ZBc<|l9;u{)*4xvA<}J5Cm{VY|5}gc~y2qb_A?w3-dUY&8XeOl9I++}lm+%#ElbE|GuO>