[DRAFT] prompt migration engine

.gitignore

@@ -3,3 +3,5 @@ __pycache__
 .ipynb_checkpoints
 wandb/
 artifacts/
+
+**/.env

recipes/use_cases/prompt-migration/.env.template

@@ -0,0 +1,5 @@
+OPENAI_API_KEY=YOUR_OPENAI_API_KEY_HERE
+REPLICATE_API_TOKEN=YOUR_REPLICATE_TOKEN_HERE
+DATABRICKS_API_TOKEN=your_databricks_token_here
+DATABRICKS_API_URL=your_databricks_endpoint_url 
+TOGETHER_API_KEY=your_together_ai_key_here

recipes/use_cases/prompt-migration/environment.yml

@@ -0,0 +1,13 @@
+name: prompt-migration
+channels:
+  - defaults
+  - pytorch
+dependencies:
+  - python=3.9
+  - pip
+  - pip:
+    - dspy-ai
+    - torch
+    - transformers
+    - openai
+    - databricks-sdk 

recipes/use_cases/prompt-migration/examples/usage.py

@@ -0,0 +1,36 @@
+import dspy
+from prompt_migration.engine import PromptMigrationEngine, PromptTemplate
+from prompt_migration.evaluator import PromptEvaluator
+
+# Initialize LMs
+openai_lm = dspy.OpenAI(model="gpt-3.5-turbo")
+target_lm = dspy.HFModel(model="gpt2")
+
+# Create migration engine
+engine = PromptMigrationEngine(openai_lm, target_lm)
+
+# Define source prompt
+source_prompt = PromptTemplate(
+    template="Summarize the following text: {text}",
+    input_variables=["text"],
+    model_type="openai"
+)
+
+eval_dataset = [
+    {"text": "Example text 1", "expected_answer": "Summary 1"},
+    {"text": "Example text 2", "expected_answer": "Summary 2"},
+]
+
+# Migrate prompt
+migrated_prompt = engine.migrate_prompt(source_prompt, eval_dataset)
+
+# Evaluate migration
+evaluator = PromptEvaluator(openai_lm, target_lm)
+metrics = evaluator.evaluate(
+    source_prompt.template,
+    migrated_prompt.template,
+    eval_dataset
+)
+
+print(f"Migrated prompt: {migrated_prompt.template}")
+print(f"Evaluation metrics: {metrics}") 

recipes/use_cases/prompt-migration/main.py

@@ -0,0 +1,103 @@
+import dspy
+from prompt_migration.engine import PromptMigrationEngine, PromptTemplate
+from prompt_migration.evaluator import PromptEvaluator
+from prompt_migration.eval_dataset import get_evaluation_dataset, get_eval_subset
+
+import os
+import dotenv
+
+dotenv.load_dotenv()
+
+def main():
+    openai_lm = dspy.LM(
+        model="gpt-3.5-turbo",
+        api_key=os.getenv("OPENAI_API_KEY")
+    )
+
+    target_lm = dspy.LM(
+        model="together_ai/meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo",
+        api_key=os.getenv("TOGETHER_API_KEY")
+    )
+    # To run it with ollama
+    # target_lm = dspy.LM('ollama_chat/llama3.2:3b-instruct-fp16', api_base='http://localhost:11434', api_key='')
+
+    # To run it with huggingface
+    # target_lm = dspy.HFModel(model="gpt2")
+
+    engine = PromptMigrationEngine(openai_lm, target_lm)
+
+    source_prompt = PromptTemplate(
+        template="""You are an advanced Large Language Model tasked with generating Python code snippets in response to user prompts. Your primary objective is to provide accurate, concise, and well-structured Python functions. Follow these guidelines:
+
+    Understand the Context: Analyze the input prompt and identify its category (e.g., API Usage, File Handling, Error Handling).
+
+    Generate Code:
+        Write Python code that directly addresses the user's request.
+        Ensure the code is syntactically correct, functional, and adheres to Python best practices.
+        Include necessary imports and handle potential edge cases.
+
+    Error Handling:
+        Include appropriate error handling where applicable (e.g., try-except blocks).
+        If exceptions occur, provide meaningful error messages.
+
+    Readability:
+        Use clear variable names and include comments where necessary for clarity.
+        Prioritize readability and maintainability in all generated code.
+
+    Complexity Alignment:
+        Tailor the code's complexity based on the indicated difficulty (e.g., simple, medium, complex).
+        Ensure that the solution is neither overly simplistic nor unnecessarily complicated.
+
+    Prompt Type:
+        Focus on the code_generation type for creating Python functions.
+        Avoid deviating from the task unless additional clarification is requested.
+
+    Testing and Validity:
+        Assume the function might be run immediately. Provide code that is ready for use or minimal adaptation.
+        Highlight any dependencies or external libraries required.
+        """,
+        input_variables=["text"],
+        model_type="openai"
+    )
+
+    eval_dataset = get_evaluation_dataset()
+
+
+    # To evaluate on a specific subset, use the following:
+    code_generation_dataset = get_eval_subset(prompt_type="code_generation")
+    #simple_tasks = get_eval_subset(complexity="simple")
+    evaluator = PromptEvaluator(openai_lm, target_lm)
+
+    metrics = evaluator.evaluate(
+        source_prompt.template,  # Same prompt for both
+        source_prompt.template,  # Same prompt for both
+        code_generation_dataset
+    )
+
+    print(f"Evaluation metrics:")
+    print(f"  Accuracy: {metrics.accuracy:.2f}")
+    print(f"  Similarity: {metrics.similarity:.2f}")
+    print(f"  Consistency: {metrics.consistency:.2f}")
+
+    # Migrate prompt
+    print("Migrating prompt...")
+    migrated_prompt = engine.migrate_prompt(source_prompt, code_generation_dataset)
+
+    # Evaluate migration
+    print("Evaluating migration...")
+    metrics = evaluator.evaluate(
+        source_prompt.template,
+        migrated_prompt.template,
+        code_generation_dataset
+    )
+
+    print(f"\nResults:")
+    print(f"Original prompt: {source_prompt.template}")
+    print(f"Migrated prompt: {migrated_prompt.template}")
+    print(f"Evaluation metrics:")
+    print(f"  Accuracy: {metrics.accuracy:.2f}")
+    print(f"  Similarity: {metrics.similarity:.2f}")
+    print(f"  Consistency: {metrics.consistency:.2f}")
+
+if __name__ == "__main__":
+    main() 

recipes/use_cases/prompt-migration/prompt_migration/engine.py

@@ -0,0 +1,106 @@
+import dspy
+from typing import List, Dict, Optional
+from dataclasses import dataclass
+
+@dataclass
+class PromptTemplate:
+    template: str
+    input_variables: List[str]
+    model_type: str  # 'openai' or 'llama'
+
+class PromptMigrationEngine:
+    def __init__(self, source_lm: dspy.LM, target_lm: dspy.LM):
+        self.source_lm = source_lm
+        self.target_lm = target_lm
+        dspy.configure(lm=source_lm)
+
+    def _optimize_transformation(self, transformer, eval_dataset):
+        """Optimize the transformation using the evaluation dataset."""
+        class PromptQualityMetric:
+            def __init__(self, source_lm, target_lm):
+                self.source_lm = source_lm
+                self.target_lm = target_lm
+
+            def __call__(self, example, prediction, trace=None):
+                if not hasattr(prediction, 'target'):
+                    return 0.0
+
+                try:
+                    # Get outputs from both models using the prompts
+                    source_output = self.source_lm(example.source)
+                    target_output = self.target_lm(prediction.target)
+
+                    # Compare outputs (basic similarity)
+                    from difflib import SequenceMatcher
+                    similarity = SequenceMatcher(None, 
+                                              str(source_output), 
+                                              str(target_output)).ratio()
+                    return similarity
+                except Exception as e:
+                    print(f"Error in metric: {e}")
+                    return 0.0
+
+        optimizer = dspy.BootstrapFewShotWithRandomSearch(
+            metric=PromptQualityMetric(self.source_lm, self.target_lm),
+            max_bootstrapped_demos=2,
+            max_labeled_demos=2,
+            num_threads=1
+        )
+
+        # Prepare training data
+        train_data = []
+        for item in eval_dataset:
+            # Create example with both prompt and expected output
+            example = dspy.Example(
+                source=item["text"],
+                expected_output=item["expected_answer"]
+            ).with_inputs("source")
+            train_data.append(example)
+
+        return optimizer.compile(transformer, trainset=train_data)
+
+    def migrate_prompt(self, 
+                      source_prompt: PromptTemplate,
+                      eval_dataset: Optional[List[Dict]] = None) -> PromptTemplate:
+        """Migrates a prompt from source LM to target LM format."""
+
+        class PromptTransformation(dspy.Signature):
+            """Convert a prompt from one format to another."""
+            source = dspy.InputField(desc="Source prompt template")
+            target = dspy.OutputField(desc="Transformed prompt template that maintains functionality while adapting to target model format")
+
+        class Transformer(dspy.Module):
+            def __init__(self):
+                super().__init__()
+                self.chain = dspy.ChainOfThought(PromptTransformation)
+
+            def forward(self, source):
+                # Add context about the transformation task
+                prompt = f"""
+                Transform this prompt while:
+                1. Maintaining core functionality
+                2. Adapting to target model format
+                3. Preserving input variables
+                4. Keeping essential instructions
+
+                Source prompt:
+                {source}
+                """
+                return self.chain(source=prompt)
+
+        transformer = Transformer()
+
+        if eval_dataset:
+            transformer = self._optimize_transformation(transformer, eval_dataset)
+
+        result = transformer(source=source_prompt.template)
+
+        # Format for target model
+        if source_prompt.model_type == "openai" and "llama" in str(self.target_lm):
+            result.target = f"### Instruction:\n{result.target}\n\n### Response:"
+
+        return PromptTemplate(
+            template=result.target,
+            input_variables=source_prompt.input_variables,
+            model_type='llama'
+        )