嵌入（Embeddings）（Embeddings）

curl https://api.openai.com/v1/embeddings
  -H "Content-Type: application/json"
  -H "Authorization: Bearer $OPENAI_API_KEY"
  -d '{
    "input": "Your text string goes here",
    "model": "text-embedding-ada-002"
  }'

{
  "data": [
    {
      "embedding": [
        -0.006929283495992422,
        -0.005336422007530928,
        ...
        -4.547132266452536e-05,
        -0.024047505110502243
      ],
      "index": 0,
      "object": "embedding"
    }
  ],
  "model": "text-embedding-ada-002",
  "object": "list",
  "usage": {
    "prompt_tokens": 5,
    "total_tokens": 5
  }
}

def get_embedding(text, model="text-embedding-ada-002"):
 text = text.replace("\n", " ")
 return openai.Embedding.create(input = [text], model=model)['data'][0]['embedding']
 
df['ada_embedding'] = df.combined.apply(lambda x: get_embedding(x, model='text-embedding-ada-002'))
df.to_csv('output/embedded_1k_reviews.csv', index=False)

import pandas as pd
 
df = pd.read_csv('output/embedded_1k_reviews.csv')
df['ada_embedding'] = df.ada_embedding.apply(eval).apply(np.array)

import pandas as pd
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
import matplotlib
 
df = pd.read_csv('output/embedded_1k_reviews.csv')
matrix = df.ada_embedding.apply(eval).to_list()
 
# Create a t-SNE model and transform the data
tsne = TSNE(n_components=2, perplexity=15, random_state=42, init='random', learning_rate=200)
vis_dims = tsne.fit_transform(matrix)
 
colors = ["red", "darkorange", "gold", "turquiose", "darkgreen"]
x = [x for x,y in vis_dims]
y = [y for x,y in vis_dims]
color_indices = df.Score.values - 1
 
colormap = matplotlib.colors.ListedColormap(colors)
plt.scatter(x, y, c=color_indices, cmap=colormap, alpha=0.3)
plt.title("Amazon ratings visualized in language using t-SNE")

from sklearn.model_selection import train_test_split
 
X_train, X_test, y_train, y_test = train_test_split(
 list(df.ada_embedding.values),
    df.Score,
 test_size = 0.2,
 random_state=42
)

from sklearn.ensemble import RandomForestRegressor
 
rfr = RandomForestRegressor(n_estimators=100)
rfr.fit(X_train, y_train)
preds = rfr.predict(X_test)

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report, accuracy_score
 
clf = RandomForestClassifier(n_estimators=100)
clf.fit(X_train, y_train)
preds = clf.predict(X_test)

from openai.embeddings_utils import cosine_similarity, get_embedding
 
df= df[df.Score!=3]
df['sentiment'] = df.Score.replace({1:'negative', 2:'negative', 4:'positive', 5:'positive'})
 
labels = ['negative', 'positive']
label_embeddings = [get_embedding(label, model=model) for label in labels]
 
def label_score(review_embedding, label_embeddings):
 return cosine_similarity(review_embedding, label_embeddings[1]) - cosine_similarity(review_embedding, label_embeddings[0])
 
prediction = 'positive' if label_score('Sample Review', label_embeddings) > 0 else 'negative'

user_embeddings = df.groupby('UserId').ada_embedding.apply(np.mean)
prod_embeddings = df.groupby('ProductId').ada_embedding.apply(np.mean)

import numpy as np
from sklearn.cluster import KMeans
 
matrix = np.vstack(df.ada_embedding.values)
n_clusters = 4
 
kmeans = KMeans(n_clusters = n_clusters, init='k-means++', random_state=42)
kmeans.fit(matrix)
df['Cluster'] = kmeans.labels_

from openai.embeddings_utils import get_embedding, cosine_similarity
 
def search_reviews(df, product_description, n=3, pprint=True):
 embedding = get_embedding(product_description, model='text-embedding-ada-002')
 df['similarities'] = df.ada_embedding.apply(lambda x: cosine_similarity(x, embedding))
 res = df.sort_values('similarities', ascending=False).head(n)
 return res
 
res = search_reviews(df, 'delicious beans', n=3)

from openai.embeddings_utils import get_embedding, cosine_similarity
 
df['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, model='text-embedding-ada-002'))
 
def search_functions(df, code_query, n=3, pprint=True, n_lines=7):
 embedding = get_embedding(code_query, model='text-embedding-ada-002')
 df['similarities'] = df.code_embedding.apply(lambda x: cosine_similarity(x, embedding))
 
 res = df.sort_values('similarities', ascending=False).head(n)
 return res
res = search_functions(df, 'Completions API tests', n=3)

def recommendations_from_strings(
 strings: List[str],
 index_of_source_string: int,
 model="text-embedding-ada-002",
) -> List[int]:
 """Return nearest neighbors of a given string."""
 
 # get embeddings for all strings
 embeddings = [embedding_from_string(string, model=model) for string in strings]
 
 # get the embedding of the source string
   query_embedding = embeddings[index_of_source_string]
 
 # get distances between the source embedding and other embeddings (function from embeddings_utils.py)
 distances = distances_from_embeddings(query_embedding, embeddings, distance_metric="cosine")
 
 # get indices of nearest neighbors (function from embeddings_utils.py)
   indices_of_nearest_neighbors = indices_of_nearest_neighbors_from_distances(distances)
 return indices_of_nearest_neighbors
 

import tiktoken
 
def num_tokens_from_string(string: str, encoding_name: str) -> int:
 """Returns the number of tokens in a text string."""
    encoding = tiktoken.get_encoding(encoding_name)
 num_tokens = len(encoding.encode(string))
 return num_tokens
 
num_tokens_from_string("tiktoken is great!", "cl100k_base")