Skip to content

Commit

Permalink
update example
Browse files Browse the repository at this point in the history
  • Loading branch information
@h0rv
h0rv committed Aug 27, 2024
2 parents 22ce522 + 6c2e9b9 commit 1375f0d
Show file tree
Hide file tree
Showing 5 changed files with 250 additions and 16 deletions.
191 changes: 184 additions & 7 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func segment(ctx context.Context, data string) *Searches {
Model: openai.GPT4o,
Messages: []openai.ChatCompletionMessage{
{
Role: instructor.RoleUser,
Role: openai.ChatMessageRoleUser,
Content: fmt.Sprintf("Consider the data below: '\n%s' and segment it into multiple search queries", data),
},
},
Expand Down Expand Up @@ -298,7 +298,7 @@ func assert(condition bool, message string) {
<details>
<summary>Images with OpenAI</summary>

![List of movies](https://raw.githubusercontent.com/instructor-ai/instructor-go/main/examples/vision/openai/books.png)
![List of books](https://raw.githubusercontent.com/instructor-ai/instructor-go/main/examples/vision/openai/books.png)

<details>
<summary>Running</summary>
Expand Down Expand Up @@ -356,7 +356,7 @@ func main() {
Model: openai.GPT4o,
Messages: []openai.ChatCompletionMessage{
{
Role: instructor.RoleUser,
Role: openai.ChatMessageRoleUser,
MultiContent: []openai.ChatMessagePart{
{
Type: openai.ChatMessagePartTypeText,
Expand Down Expand Up @@ -437,7 +437,7 @@ func main() {
<details>
<summary>Images with Anthropic</summary>

![List of books](https://raw.githubusercontent.com/instructor-ai/instructor-go/main/examples/vision/anthropic/movies.png)
![List of movies](https://raw.githubusercontent.com/instructor-ai/instructor-go/main/examples/vision/anthropic/movies.png)

<details>
<summary>Running</summary>
Expand Down Expand Up @@ -504,7 +504,7 @@ func main() {
Model: "claude-3-haiku-20240307",
Messages: []anthropic.Message{
{
Role: instructor.RoleUser,
Role: anthropic.RoleUser,
Content: []anthropic.MessageContent{
anthropic.NewImageMessageContent(anthropic.MessageContentImageSource{
Type: "base64",
Expand Down Expand Up @@ -696,15 +696,15 @@ Preferred Shopping Times: Weekend Evenings
Model: openai.GPT4o20240513,
Messages: []openai.ChatCompletionMessage{
{
Role: instructor.RoleSystem,
Role: openai.ChatMessageRoleSystem,
Content: fmt.Sprintf(`
Generate the product recommendations from the product list based on the customer profile.
Return in order of highest recommended first.
Product list:
%s`, productList),
},
{
Role: instructor.RoleUser,
Role: openai.ChatMessageRoleUser,
Content: fmt.Sprintf("User profile:\n%s", profileData),
},
},
Expand Down Expand Up @@ -1090,6 +1090,183 @@ func main() {
```
</details>

<details>
<summary>Receipt Item Extraction from Image (using OpenAI GPT-4o)</summary>

<p align="center">
<img src="https://raw.githubusercontent.com/instructor-ai/instructor-go/main/examples/vision/receipt/supermarket-receipt-template.jpg" alt="Receipt 1" width="45%">
<img src="https://raw.githubusercontent.com/instructor-ai/instructor-go/main/examples/vision/receipt/receipt-ocr-original.jpg" alt="Receipt 2" width="45%">
</p>

<details>
<summary>Running</summary>

```bash
go run examples/vision/receipt/main.go
```

</details>

```go
package main

import (
"context"
"fmt"
"math"
"os"

"github.com/instructor-ai/instructor-go/pkg/instructor"
openai "github.com/sashabaranov/go-openai"
)

type Item struct {
Name string `json:"name" jsonschema:"title=Item Name,description=The name of the item,example=Apple,example=Banana"`
Price float64 `json:"price" jsonschema:"title=Item Price,description=The price of the item in dollars,example=1.99,example=2.50"`
}

func (i Item) String() string {
return fmt.Sprintf(" Item: %s, Price: $%.2f", i.Name, i.Price)
}

type Receipt struct {
Items []Item `json:"items" jsonschema:"title=Receipt Items,description=The list of items in the receipt"`
Total float64 `json:"total" jsonschema:"title=Receipt Total,description=The total cost of all items in the receipt,example=10.99,example=25.50"`
}

func (r Receipt) String() string {
var result string
for _, item := range r.Items {
result += item.String() + "\n"
}
result += fmt.Sprintf("Total: $%.2f", r.Total)
return result
}

func (r *Receipt) Validate() error {
calculatedTotal := 0.0
for _, item := range r.Items {
calculatedTotal += item.Price
}

calculatedTotal = math.Round(calculatedTotal*10) / 10
expectedTotal := math.Round(r.Total*10) / 10

if calculatedTotal != expectedTotal {
return fmt.Errorf("total %f does not match the sum of item prices %f", r.Total, calculatedTotal)
}
return nil
}

func extract(ctx context.Context, client *instructor.InstructorOpenAI, url string) (*Receipt, error) {

var receipt Receipt
_, err := client.CreateChatCompletion(
ctx,
openai.ChatCompletionRequest{
Model: openai.GPT4o,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
Content: `Analyze the image and return the items (include tax and coupons as their own items) in the receipt and the total amount.`,
},
{
Role: openai.ChatMessageRoleUser,
MultiContent: []openai.ChatMessagePart{
{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
URL: url,
},
},
},
},
},
},
&receipt,
)
if err != nil {
return nil, err
}

if err := receipt.Validate(); err != nil {
return &receipt, err
}

return &receipt, nil
}

func main() {
ctx := context.Background()

client := instructor.FromOpenAI(
openai.NewClient(os.Getenv("OPENAI_API_KEY")),
instructor.WithMode(instructor.ModeJSON),
instructor.WithMaxRetries(3),
)

urls := []string{
"https://templates.mediamodifier.com/645124ff36ed2f5227cbf871/supermarket-receipt-template.jpg",
"https://ocr.space/Content/Images/receipt-ocr-original.jpg",
}

for _, url := range urls {
receipt, err := extract(ctx, client, url)
fmt.Printf("Receipt:\n%s\n", receipt)
if err != nil {
fmt.Printf("Error: %v\n", err)
continue
}
fmt.Println("\n--------------------------------\n")
}
/*
Receipt:
Item: Lorem ipsum, Price: $9.20
Item: Lorem ipsum dolor sit, Price: $19.20
Item: Lorem ipsum dolor sit amet, Price: $15.00
Item: Lorem ipsum, Price: $15.00
Item: Lorem ipsum, Price: $15.00
Item: Lorem ipsum dolor sit, Price: $15.00
Item: Lorem ipsum, Price: $19.20
Total: $107.60
--------------------------------
Receipt:
Item: PET TOY, Price: $1.97
Item: FLOPPY PUPPY, Price: $1.97
Item: SSSUPREME S, Price: $4.97
Item: 2.5 SQUEAK, Price: $5.92
Item: MUNCHY DMBEL, Price: $3.77
Item: DOG TREAT, Price: $2.92
Item: PED PCH 1, Price: $0.50
Item: PED PCH 1, Price: $0.50
Item: HNYMD SMORES, Price: $3.98
Item: FRENCH DRSNG, Price: $1.98
Item: 3 ORANGES, Price: $5.47
Item: BABY CARROTS, Price: $1.48
Item: COLLARDS, Price: $1.24
Item: CALZONE, Price: $2.50
Item: MM RVW MNT, Price: $19.77
Item: STKOBRLPLIABL, Price: $1.97
Item: STKOBRLPLIABL, Price: $1.97
Item: STKO SUNFLWR, Price: $0.97
Item: STKO SUNFLWR, Price: $0.97
Item: STKO SUNFLWR, Price: $0.97
Item: STKO SUNFLWR, Price: $0.97
Item: BLING BEADS, Price: $0.97
Item: GREAT VALUE, Price: $9.97
Item: LIPTON, Price: $4.44
Item: DRY DOG, Price: $12.44
Item: COUPON 2310652, Price: $-1.00
Item: TAX, Price: $4.59
Total: $98.21
*/
}
```

</details>

## Providers

Instructor Go supports the following LLM provider APIs:
Expand Down
6 changes: 6 additions & 0 deletions examples/vision/receipt/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Receipt Image Extractor

<p align="center">
<img src="https://raw.githubusercontent.com/instructor-ai/instructor-go/main/examples/vision/receipt/supermarket-receipt-template.jpg" alt="Receipt 1" width="45%">
<img src="https://raw.githubusercontent.com/instructor-ai/instructor-go/main/examples/vision/receipt/receipt-ocr-original.jpg" alt="Receipt 2" width="45%">
</p>
69 changes: 60 additions & 9 deletions examples/vision/receipt/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@ type Receipt struct {
Total float64 `json:"total"`
}

// Validate method similar to the Pydantic model validator
func (r *Receipt) String() string {
s, _ := json.MarshalIndent(r, "", " ")
return string(s)
}

func (r *Receipt) Validate() error {
calculatedTotal := 0.0
for _, item := range r.Items {
Expand All @@ -33,7 +37,6 @@ func (r *Receipt) Validate() error {
return nil
}

// Function to extract receipt information from a URL
func extract(ctx context.Context, client *instructor.InstructorOpenAI, url string) (*Receipt, error) {
var receipt Receipt

Expand All @@ -58,7 +61,6 @@ func extract(ctx context.Context, client *instructor.InstructorOpenAI, url strin
return nil, err
}

// Validate the receipt total
if err := receipt.Validate(); err != nil {
return &receipt, err
}
Expand All @@ -76,17 +78,66 @@ func main() {
)

urls := []string{
"https://templates.mediamodifier.com/645124ff36ed2f5227cbf871/supermarket-receipt-template.jpg",
"https://ocr.space/Content/Images/receipt-ocr-original.jpg",
// source: https://templates.mediamodifier.com/645124ff36ed2f5227cbf871/supermarket-receipt-template.jpg
"https://raw.githubusercontent.com/instructor-ai/instructor-go/main/examples/vision/receipt/supermarket-receipt-template.jpg",
// source: https://ocr.space/Content/Images/receipt-ocr-original.jpg
"https://raw.githubusercontent.com/instructor-ai/instructor-go/main/examples/vision/receipt/receipt-ocr-original.jpg",
}

for _, url := range urls {
receipt, err := extract(ctx, client, url)
println("Receipt: ")
println(receipt.String())
if err != nil {
fmt.Printf("Error: %v\n", err)
continue
println("Error: " + err.Error())
}
receiptJson, _ := json.MarshalIndent(receipt, "", " ")
fmt.Printf("Receipt: %s\n", receiptJson)
println("\n--------------------------------\n")
}
/*
Receipt:
Item: Lorem ipsum, Price: $9.20
Item: Lorem ipsum dolor sit, Price: $19.20
Item: Lorem ipsum dolor sit amet, Price: $15.00
Item: Lorem ipsum, Price: $15.00
Item: Lorem ipsum, Price: $15.00
Item: Lorem ipsum dolor sit, Price: $15.00
Item: Lorem ipsum, Price: $19.20
Total: $107.60
--------------------------------
Receipt:
Item: PET TOY, Price: $1.97
Item: FLOPPY PUPPY, Price: $1.97
Item: SSSUPREME S, Price: $4.97
Item: 2.5 SQUEAK, Price: $5.92
Item: MUNCHY DMBEL, Price: $3.77
Item: DOG TREAT, Price: $2.92
Item: PED PCH 1, Price: $0.50
Item: PED PCH 1, Price: $0.50
Item: HNYMD SMORES, Price: $3.98
Item: FRENCH DRSNG, Price: $1.98
Item: 3 ORANGES, Price: $5.47
Item: BABY CARROTS, Price: $1.48
Item: COLLARDS, Price: $1.24
Item: CALZONE, Price: $2.50
Item: MM RVW MNT, Price: $19.77
Item: STKOBRLPLIABL, Price: $1.97
Item: STKOBRLPLIABL, Price: $1.97
Item: STKO SUNFLWR, Price: $0.97
Item: STKO SUNFLWR, Price: $0.97
Item: STKO SUNFLWR, Price: $0.97
Item: STKO SUNFLWR, Price: $0.97
Item: BLING BEADS, Price: $0.97
Item: GREAT VALUE, Price: $9.97
Item: LIPTON, Price: $4.44
Item: DRY DOG, Price: $12.44
Item: COUPON 2310652, Price: $-1.00
Item: TAX, Price: $4.59
Total: $98.21
*/
}
Binary file added examples/vision/receipt/receipt-ocr-original.jpg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added examples/vision/receipt/supermarket-receipt-template.jpg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit 1375f0d

Please sign in to comment.