changes to imports & import Embeddings

This commit is contained in:
2024-07-12 22:00:29 +02:00
parent b4644ea295
commit bf4bd69337
20 changed files with 751 additions and 140 deletions

View File

@@ -1,15 +1,30 @@
import 'dotenv/config';
import { drizzle } from 'drizzle-orm/node-postgres';
import { existsSync, readFileSync, readdirSync, statSync, unlinkSync } from 'fs';
import { existsSync, readdirSync, readFileSync, statSync, unlinkSync } from 'fs';
import fs from 'fs-extra';
import OpenAI from 'openai';
import { join } from 'path';
import pkg from 'pg';
import { rimraf } from 'rimraf';
import sharp from 'sharp';
import { BusinessListing, CommercialPropertyListing, User, UserData } from 'src/models/db.model.js';
import { emailToDirName } from 'src/models/main.model.js';
import { BusinessListing, CommercialPropertyListing, User, UserData } from '../models/db.model.js';
import { emailToDirName, KeyValueStyle } from '../models/main.model.js';
import * as schema from './schema.js';
const typesOfBusiness: Array<KeyValueStyle> = [
{ name: 'Automotive', value: '1', icon: 'fa-solid fa-car', textColorClass: 'text-green-400' },
{ name: 'Industrial Services', value: '2', icon: 'fa-solid fa-industry', textColorClass: 'text-yellow-400' },
{ name: 'Real Estate', value: '3', icon: 'fa-solid fa-building', textColorClass: 'text-blue-400' },
{ name: 'Uncategorized', value: '4', icon: 'fa-solid fa-question', textColorClass: 'text-cyan-400' },
{ name: 'Retail', value: '5', icon: 'fa-solid fa-money-bill-wave', textColorClass: 'text-pink-400' },
{ name: 'Oilfield SVE and MFG.', value: '6', icon: 'fa-solid fa-oil-well', textColorClass: 'text-indigo-400' },
{ name: 'Service', value: '7', icon: 'fa-solid fa-umbrella', textColorClass: 'text-teal-400' },
{ name: 'Advertising', value: '8', icon: 'fa-solid fa-rectangle-ad', textColorClass: 'text-orange-400' },
{ name: 'Agriculture', value: '9', icon: 'fa-solid fa-wheat-awn', textColorClass: 'text-sky-400' },
{ name: 'Franchise', value: '10', icon: 'fa-solid fa-star', textColorClass: 'text-purple-400' },
{ name: 'Professional', value: '11', icon: 'fa-solid fa-user-gear', textColorClass: 'text-gray-400' },
{ name: 'Manufacturing', value: '12', icon: 'fa-solid fa-industry', textColorClass: 'text-red-400' },
{ name: 'Food and Restaurant', value: '13', icon: 'fa-solid fa-utensils', textColorClass: 'text-amber-700' },
];
const { Pool } = pkg;
const openai = new OpenAI({
@@ -88,11 +103,11 @@ for (const userData of usersData) {
const data = readFileSync(`./pictures_base/logo/${i}.jpg`);
await storeCompanyLogo(data, emailToDirName(u[0].email));
}
//Business Listings
filePath = `./data/businesses.json`;
data = readFileSync(filePath, 'utf8');
const businessJsonData = JSON.parse(data) as BusinessListing[]; // Erwartet ein Array von Objekten
for (const business of businessJsonData) {
delete business.id;
business.created = new Date(business.created);
@@ -100,7 +115,28 @@ for (const business of businessJsonData) {
const user = getRandomItem(generatedUserData);
business.userId = user.insertedId;
business.imageName = emailToDirName(user.email);
await db.insert(schema.businesses).values(business);
const embeddingText = JSON.stringify({
type: typesOfBusiness.find(b => b.value === String(business.type))?.name,
title: business.title,
description: business.description,
city: business.city,
state: business.state,
price: business.price,
realEstateIncluded: business.realEstateIncluded,
leasedLocation: business.leasedLocation,
franchiseResale: business.franchiseResale,
salesRevenue: business.salesRevenue,
cashFlow: business.cashFlow,
supportAndTraining: business.supportAndTraining,
employees: business.employees,
established: business.established,
reasonForSale: business.reasonForSale,
});
const embedding = await createEmbedding(embeddingText);
await db.insert(schema.businesses).values({
...business,
embedding: embedding,
});
}
//Corporate Listings
filePath = `./data/commercials.json`;
@@ -131,7 +167,7 @@ await client.end();
async function createEmbedding(text: string): Promise<number[]> {
const response = await openai.embeddings.create({
model: 'text-embedding-ada-002',
model: 'text-embedding-3-small',
input: text,
});
return response.data[0].embedding;