aknuth/bizmatch-project
1
0
Fork 0
Code Issues 9 Pull Requests Packages Projects 5 Releases Wiki Activity

Issue: #109

Browse Source
This commit is contained in:
Andreas Knuth
2024-09-10 21:30:03 +02:00
parent 17213ba4b0
commit 83307684ee
25 changed files with 799 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

431
bizmatch-server/src/drizzle/import.ts
View File

@@ -2,19 +2,18 @@ import 'dotenv/config';
import { drizzle } from 'drizzle-orm/node-postgres';
import { existsSync, readdirSync, readFileSync, statSync, unlinkSync } from 'fs';
import fs from 'fs-extra';
import OpenAI from 'openai';
import { join } from 'path';
import pkg from 'pg';
import { Pool } from 'pg';
import { rimraf } from 'rimraf';
import sharp from 'sharp';
import { BusinessListingService } from 'src/listings/business-listing.service';
import { CommercialPropertyService } from 'src/listings/commercial-property.service';
import { Geo } from 'src/models/server.model';
import { UserService } from 'src/user/user.service';
import winston from 'winston';
import { User, UserData } from '../models/db.model';
import { createDefaultBusinessListing, createDefaultCommercialPropertyListing, createDefaultUser, emailToDirName, KeyValueStyle } from '../models/main.model';
import { createDefaultBusinessListing, createDefaultCommercialPropertyListing, createDefaultUser, emailToDirName } from '../models/main.model';
import { SelectOptionsService } from '../select-options/select-options.service';
import { convertUserToDrizzleUser } from '../utils';
import * as schema from './schema';
interface PropertyImportListing {
id: string;
@@ -54,224 +53,226 @@ interface BusinessImportListing {
internals: string;
created: string;
}
const typesOfBusiness: Array<KeyValueStyle> = [
{ name: 'Automotive', value: '1', icon: 'fa-solid fa-car', textColorClass: 'text-green-400' },
{ name: 'Industrial Services', value: '2', icon: 'fa-solid fa-industry', textColorClass: 'text-yellow-400' },
{ name: 'Real Estate', value: '3', icon: 'fa-solid fa-building', textColorClass: 'text-blue-400' },
{ name: 'Uncategorized', value: '4', icon: 'fa-solid fa-question', textColorClass: 'text-cyan-400' },
{ name: 'Retail', value: '5', icon: 'fa-solid fa-money-bill-wave', textColorClass: 'text-pink-400' },
{ name: 'Oilfield SVE and MFG.', value: '6', icon: 'fa-solid fa-oil-well', textColorClass: 'text-indigo-400' },
{ name: 'Service', value: '7', icon: 'fa-solid fa-umbrella', textColorClass: 'text-teal-400' },
{ name: 'Advertising', value: '8', icon: 'fa-solid fa-rectangle-ad', textColorClass: 'text-orange-400' },
{ name: 'Agriculture', value: '9', icon: 'fa-solid fa-wheat-awn', textColorClass: 'text-sky-400' },
{ name: 'Franchise', value: '10', icon: 'fa-solid fa-star', textColorClass: 'text-purple-400' },
{ name: 'Professional', value: '11', icon: 'fa-solid fa-user-gear', textColorClass: 'text-gray-400' },
{ name: 'Manufacturing', value: '12', icon: 'fa-solid fa-industry', textColorClass: 'text-red-400' },
{ name: 'Food and Restaurant', value: '13', icon: 'fa-solid fa-utensils', textColorClass: 'text-amber-700' },
];
const { Pool } = pkg;
// const typesOfBusiness: Array<KeyValueStyle> = [
// { name: 'Automotive', value: '1', icon: 'fa-solid fa-car', textColorClass: 'text-green-400' },
// { name: 'Industrial Services', value: '2', icon: 'fa-solid fa-industry', textColorClass: 'text-yellow-400' },
// { name: 'Real Estate', value: '3', icon: 'fa-solid fa-building', textColorClass: 'text-blue-400' },
// { name: 'Uncategorized', value: '4', icon: 'fa-solid fa-question', textColorClass: 'text-cyan-400' },
// { name: 'Retail', value: '5', icon: 'fa-solid fa-money-bill-wave', textColorClass: 'text-pink-400' },
// { name: 'Oilfield SVE and MFG.', value: '6', icon: 'fa-solid fa-oil-well', textColorClass: 'text-indigo-400' },
// { name: 'Service', value: '7', icon: 'fa-solid fa-umbrella', textColorClass: 'text-teal-400' },
// { name: 'Advertising', value: '8', icon: 'fa-solid fa-rectangle-ad', textColorClass: 'text-orange-400' },
// { name: 'Agriculture', value: '9', icon: 'fa-solid fa-wheat-awn', textColorClass: 'text-sky-400' },
// { name: 'Franchise', value: '10', icon: 'fa-solid fa-star', textColorClass: 'text-purple-400' },
// { name: 'Professional', value: '11', icon: 'fa-solid fa-user-gear', textColorClass: 'text-gray-400' },
// { name: 'Manufacturing', value: '12', icon: 'fa-solid fa-industry', textColorClass: 'text-red-400' },
// { name: 'Food and Restaurant', value: '13', icon: 'fa-solid fa-utensils', textColorClass: 'text-amber-700' },
// ];
// const { Pool } = pkg;
const openai = new OpenAI({
apiKey: process.env.OPENAI_API_KEY, // Stellen Sie sicher, dass Sie Ihren API-Key als Umgebungsvariable setzen
});
const connectionString = process.env.DATABASE_URL;
// const pool = new Pool({connectionString})
const client = new Pool({ connectionString });
const db = drizzle(client, { schema, logger: true });
const logger = winston.createLogger({
transports: [new winston.transports.Console()],
});
const commService = new CommercialPropertyService(null, db);
const businessService = new BusinessListingService(null, db);
//Delete Content
await db.delete(schema.commercials);
await db.delete(schema.businesses);
await db.delete(schema.users);
let filePath = `./src/assets/geo.json`;
const rawData = readFileSync(filePath, 'utf8');
const geos = JSON.parse(rawData) as Geo;
const sso = new SelectOptionsService();
//Broker
filePath = `./data/broker.json`;
let data: string = readFileSync(filePath, 'utf8');
const usersData: UserData[] = JSON.parse(data); // Erwartet ein Array von Objekten
const generatedUserData = [];
console.log(usersData.length);
let i = 0,
male = 0,
female = 0;
const targetPathProfile = `./pictures/profile`;
deleteFilesOfDir(targetPathProfile);
const targetPathLogo = `./pictures/logo`;
deleteFilesOfDir(targetPathLogo);
const targetPathProperty = `./pictures/property`;
deleteFilesOfDir(targetPathProperty);
fs.ensureDirSync(`./pictures/logo`);
fs.ensureDirSync(`./pictures/profile`);
fs.ensureDirSync(`./pictures/property`);
//User
for (let index = 0; index < usersData.length; index++) {
const userData = usersData[index];
const user: User = createDefaultUser('', '', '', null);
user.licensedIn = [];
userData.licensedIn.forEach(l => {
console.log(l['value'], l['name']);
user.licensedIn.push({ registerNo: l['value'], state: l['name'] });
// const openai = new OpenAI({
// apiKey: process.env.OPENAI_API_KEY, // Stellen Sie sicher, dass Sie Ihren API-Key als Umgebungsvariable setzen
// });
(async () => {
const connectionString = process.env.DATABASE_URL;
// const pool = new Pool({connectionString})
const client = new Pool({ connectionString });
const db = drizzle(client, { schema, logger: true });
const logger = winston.createLogger({
transports: [new winston.transports.Console()],
});
user.areasServed = [];
user.areasServed = userData.areasServed.map(l => {
return { county: l.split(',')[0].trim(), state: l.split(',')[1].trim() };
});
user.hasCompanyLogo = true;
user.hasProfile = true;
user.firstname = userData.firstname;
user.lastname = userData.lastname;
user.email = userData.email;
user.phoneNumber = userData.phoneNumber;
user.description = userData.description;
user.companyName = userData.companyName;
user.companyOverview = userData.companyOverview;
user.companyWebsite = userData.companyWebsite;
const [city, state] = userData.companyLocation.split('-').map(e => e.trim());
user.companyLocation = {};
user.companyLocation.name = city;
user.companyLocation.state = state;
const cityGeo = geos.states.find(s => s.state_code === state).cities.find(c => c.name === city);
user.companyLocation.latitude = cityGeo.latitude;
user.companyLocation.longitude = cityGeo.longitude;
user.offeredServices = userData.offeredServices;
user.gender = userData.gender;
user.customerType = 'professional';
user.customerSubType = 'broker';
user.created = new Date();
user.updated = new Date();
const commService = new CommercialPropertyService(null, db);
const businessService = new BusinessListingService(null, db);
const userService = new UserService(null, db, null, null);
//Delete Content
await db.delete(schema.commercials);
await db.delete(schema.businesses);
await db.delete(schema.users);
let filePath = `./src/assets/geo.json`;
const rawData = readFileSync(filePath, 'utf8');
const geos = JSON.parse(rawData) as Geo;
const u = await db
.insert(schema.users)
.values(convertUserToDrizzleUser(user))
.returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email, firstname: schema.users.firstname, lastname: schema.users.lastname });
generatedUserData.push(u[0]);
i++;
logger.info(`user_${index} inserted`);
if (u[0].gender === 'male') {
male++;
const data = readFileSync(`./pictures_base/profile/Mann_${male}.jpg`);
await storeProfilePicture(data, emailToDirName(u[0].email));
} else {
female++;
const data = readFileSync(`./pictures_base/profile/Frau_${female}.jpg`);
await storeProfilePicture(data, emailToDirName(u[0].email));
const sso = new SelectOptionsService();
//Broker
filePath = `./data/broker.json`;
let data: string = readFileSync(filePath, 'utf8');
const usersData: UserData[] = JSON.parse(data); // Erwartet ein Array von Objekten
const generatedUserData = [];
console.log(usersData.length);
let i = 0,
male = 0,
female = 0;
const targetPathProfile = `./pictures/profile`;
deleteFilesOfDir(targetPathProfile);
const targetPathLogo = `./pictures/logo`;
deleteFilesOfDir(targetPathLogo);
const targetPathProperty = `./pictures/property`;
deleteFilesOfDir(targetPathProperty);
fs.ensureDirSync(`./pictures/logo`);
fs.ensureDirSync(`./pictures/profile`);
fs.ensureDirSync(`./pictures/property`);
//User
for (let index = 0; index < usersData.length; index++) {
const userData = usersData[index];
const user: User = createDefaultUser('', '', '', null);
user.licensedIn = [];
userData.licensedIn.forEach(l => {
console.log(l['value'], l['name']);
user.licensedIn.push({ registerNo: l['value'], state: l['name'] });
});
user.areasServed = [];
user.areasServed = userData.areasServed.map(l => {
return { county: l.split(',')[0].trim(), state: l.split(',')[1].trim() };
});
user.hasCompanyLogo = true;
user.hasProfile = true;
user.firstname = userData.firstname;
user.lastname = userData.lastname;
user.email = userData.email;
user.phoneNumber = userData.phoneNumber;
user.description = userData.description;
user.companyName = userData.companyName;
user.companyOverview = userData.companyOverview;
user.companyWebsite = userData.companyWebsite;
const [city, state] = userData.companyLocation.split('-').map(e => e.trim());
user.location = {};
user.location.name = city;
user.location.state = state;
const cityGeo = geos.states.find(s => s.state_code === state).cities.find(c => c.name === city);
user.location.latitude = cityGeo.latitude;
user.location.longitude = cityGeo.longitude;
user.offeredServices = userData.offeredServices;
user.gender = userData.gender;
user.customerType = 'professional';
user.customerSubType = 'broker';
user.created = new Date();
user.updated = new Date();
// const u = await db
// .insert(schema.users)
// .values(convertUserToDrizzleUser(user))
// .returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email, firstname: schema.users.firstname, lastname: schema.users.lastname });
const u = await userService.saveUser(user);
generatedUserData.push(u);
i++;
logger.info(`user_${index} inserted`);
if (u.gender === 'male') {
male++;
const data = readFileSync(`./pictures_base/profile/Mann_${male}.jpg`);
await storeProfilePicture(data, emailToDirName(u.email));
} else {
female++;
const data = readFileSync(`./pictures_base/profile/Frau_${female}.jpg`);
await storeProfilePicture(data, emailToDirName(u.email));
}
const data = readFileSync(`./pictures_base/logo/${i}.jpg`);
await storeCompanyLogo(data, emailToDirName(u.email));
}
const data = readFileSync(`./pictures_base/logo/${i}.jpg`);
await storeCompanyLogo(data, emailToDirName(u[0].email));
}
//Corporate Listings
filePath = `./data/commercials.json`;
data = readFileSync(filePath, 'utf8');
const commercialJsonData = JSON.parse(data) as PropertyImportListing[]; // Erwartet ein Array von Objekten
for (let index = 0; index < commercialJsonData.length; index++) {
const user = getRandomItem(generatedUserData);
const commercial = createDefaultCommercialPropertyListing();
const id = commercialJsonData[index].id;
delete commercial.id;
//Corporate Listings
filePath = `./data/commercials.json`;
data = readFileSync(filePath, 'utf8');
const commercialJsonData = JSON.parse(data) as PropertyImportListing[]; // Erwartet ein Array von Objekten
for (let index = 0; index < commercialJsonData.length; index++) {
const user = getRandomItem(generatedUserData);
const commercial = createDefaultCommercialPropertyListing();
const id = commercialJsonData[index].id;
delete commercial.id;
commercial.email = user.email;
commercial.type = sso.typesOfCommercialProperty.find(e => e.oldValue === String(commercialJsonData[index].type)).value;
commercial.title = commercialJsonData[index].title;
commercial.description = commercialJsonData[index].description;
try {
const cityGeo = geos.states.find(s => s.state_code === commercialJsonData[index].state).cities.find(c => c.name === commercialJsonData[index].city);
commercial.location = {};
commercial.location.latitude = cityGeo.latitude;
commercial.location.longitude = cityGeo.longitude;
commercial.location.name = commercialJsonData[index].city;
commercial.location.state = commercialJsonData[index].state;
// console.log(JSON.stringify(commercial.location));
} catch (e) {
console.log(`----------------> ERROR ${commercialJsonData[index].state} - ${commercialJsonData[index].city}`);
continue;
commercial.email = user.email;
commercial.type = sso.typesOfCommercialProperty.find(e => e.oldValue === String(commercialJsonData[index].type)).value;
commercial.title = commercialJsonData[index].title;
commercial.description = commercialJsonData[index].description;
try {
const cityGeo = geos.states.find(s => s.state_code === commercialJsonData[index].state).cities.find(c => c.name === commercialJsonData[index].city);
commercial.location = {};
commercial.location.latitude = cityGeo.latitude;
commercial.location.longitude = cityGeo.longitude;
commercial.location.name = commercialJsonData[index].city;
commercial.location.state = commercialJsonData[index].state;
// console.log(JSON.stringify(commercial.location));
} catch (e) {
console.log(`----------------> ERROR ${commercialJsonData[index].state} - ${commercialJsonData[index].city}`);
continue;
}
commercial.price = commercialJsonData[index].price;
commercial.listingsCategory = 'commercialProperty';
commercial.draft = false;
commercial.imageOrder = getFilenames(id);
commercial.imagePath = emailToDirName(user.email);
const insertionDate = getRandomDateWithinLastYear();
commercial.created = insertionDate;
commercial.updated = insertionDate;
const result = await commService.createListing(commercial); //await db.insert(schema.commercials).values(commercial).returning();
try {
fs.copySync(`./pictures_base/property/${id}`, `./pictures/property/${result.imagePath}/${result.serialId}`);
} catch (err) {
console.log(`----- No pictures available for ${id} ------ ${err}`);
}
}
commercial.price = commercialJsonData[index].price;
commercial.listingsCategory = 'commercialProperty';
commercial.draft = false;
commercial.imageOrder = getFilenames(id);
commercial.imagePath = emailToDirName(user.email);
const insertionDate = getRandomDateWithinLastYear();
commercial.created = insertionDate;
commercial.updated = insertionDate;
const result = await commService.createListing(commercial); //await db.insert(schema.commercials).values(commercial).returning();
try {
fs.copySync(`./pictures_base/property/${id}`, `./pictures/property/${result.imagePath}/${result.serialId}`);
} catch (err) {
console.log(`----- No pictures available for ${id} ------ ${err}`);
//Business Listings
filePath = `./data/businesses.json`;
data = readFileSync(filePath, 'utf8');
const businessJsonData = JSON.parse(data) as BusinessImportListing[]; // Erwartet ein Array von Objekten
for (let index = 0; index < businessJsonData.length; index++) {
const business = createDefaultBusinessListing(); //businessJsonData[index];
delete business.id;
const user = getRandomItem(generatedUserData);
business.email = user.email;
business.type = sso.typesOfBusiness.find(e => e.oldValue === String(businessJsonData[index].type)).value;
business.title = businessJsonData[index].title;
business.description = businessJsonData[index].description;
try {
const cityGeo = geos.states.find(s => s.state_code === businessJsonData[index].state).cities.find(c => c.name === businessJsonData[index].city);
business.location = {};
business.location.latitude = cityGeo.latitude;
business.location.longitude = cityGeo.longitude;
business.location.name = businessJsonData[index].city;
business.location.state = businessJsonData[index].state;
} catch (e) {
console.log(`----------------> ERROR ${businessJsonData[index].state} - ${businessJsonData[index].city}`);
continue;
}
business.price = businessJsonData[index].price;
business.title = businessJsonData[index].title;
business.draft = businessJsonData[index].draft;
business.listingsCategory = 'business';
business.realEstateIncluded = businessJsonData[index].realEstateIncluded;
business.leasedLocation = businessJsonData[index].leasedLocation;
business.franchiseResale = businessJsonData[index].franchiseResale;
business.salesRevenue = businessJsonData[index].salesRevenue;
business.cashFlow = businessJsonData[index].cashFlow;
business.supportAndTraining = businessJsonData[index].supportAndTraining;
business.employees = businessJsonData[index].employees;
business.established = businessJsonData[index].established;
business.internalListingNumber = businessJsonData[index].internalListingNumber;
business.reasonForSale = businessJsonData[index].reasonForSale;
business.brokerLicencing = businessJsonData[index].brokerLicencing;
business.internals = businessJsonData[index].internals;
business.imageName = emailToDirName(user.email);
business.created = new Date(businessJsonData[index].created);
business.updated = new Date(businessJsonData[index].created);
await businessService.createListing(business); //db.insert(schema.businesses).values(business);
}
}
//Business Listings
filePath = `./data/businesses.json`;
data = readFileSync(filePath, 'utf8');
const businessJsonData = JSON.parse(data) as BusinessImportListing[]; // Erwartet ein Array von Objekten
for (let index = 0; index < businessJsonData.length; index++) {
const business = createDefaultBusinessListing(); //businessJsonData[index];
delete business.id;
const user = getRandomItem(generatedUserData);
business.email = user.email;
business.type = sso.typesOfBusiness.find(e => e.oldValue === String(businessJsonData[index].type)).value;
business.title = businessJsonData[index].title;
business.description = businessJsonData[index].description;
try {
const cityGeo = geos.states.find(s => s.state_code === businessJsonData[index].state).cities.find(c => c.name === businessJsonData[index].city);
business.location = {};
business.location.latitude = cityGeo.latitude;
business.location.longitude = cityGeo.longitude;
business.location.name = businessJsonData[index].city;
business.location.state = businessJsonData[index].state;
} catch (e) {
console.log(`----------------> ERROR ${businessJsonData[index].state} - ${businessJsonData[index].city}`);
continue;
}
business.price = businessJsonData[index].price;
business.title = businessJsonData[index].title;
business.draft = businessJsonData[index].draft;
business.listingsCategory = 'business';
business.realEstateIncluded = businessJsonData[index].realEstateIncluded;
business.leasedLocation = businessJsonData[index].leasedLocation;
business.franchiseResale = businessJsonData[index].franchiseResale;
business.salesRevenue = businessJsonData[index].salesRevenue;
business.cashFlow = businessJsonData[index].cashFlow;
business.supportAndTraining = businessJsonData[index].supportAndTraining;
business.employees = businessJsonData[index].employees;
business.established = businessJsonData[index].established;
business.internalListingNumber = businessJsonData[index].internalListingNumber;
business.reasonForSale = businessJsonData[index].reasonForSale;
business.brokerLicencing = businessJsonData[index].brokerLicencing;
business.internals = businessJsonData[index].internals;
business.imageName = emailToDirName(user.email);
business.created = new Date(businessJsonData[index].created);
business.updated = new Date(businessJsonData[index].created);
await businessService.createListing(business); //db.insert(schema.businesses).values(business);
}
//End
await client.end();
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async function createEmbedding(text: string): Promise<number[]> {
const response = await openai.embeddings.create({
model: 'text-embedding-3-small',
input: text,
});
return response.data[0].embedding;
}
//End
await client.end();
})();
// function sleep(ms) {
// return new Promise(resolve => setTimeout(resolve, ms));
// }
// async function createEmbedding(text: string): Promise<number[]> {
// const response = await openai.embeddings.create({
// model: 'text-embedding-3-small',
// input: text,
// });
// return response.data[0].embedding;
// }
function getRandomItem<T>(arr: T[]): T {
if (arr.length === 0) {
@@ -283,7 +284,7 @@ function getRandomItem<T>(arr: T[]): T {
}
function getFilenames(id: string): string[] {
try {
let filePath = `./pictures_base/property/${id}`;
const filePath = `./pictures_base/property/${id}`;
return readdirSync(filePath);
} catch (e) {
return [];
@@ -300,7 +301,7 @@ function getRandomDateWithinLastYear(): Date {
return randomDate;
}
async function storeProfilePicture(buffer: Buffer, userId: string) {
let quality = 50;
const quality = 50;
const output = await sharp(buffer)
.resize({ width: 300 })
.avif({ quality }) // Verwende AVIF
@@ -310,7 +311,7 @@ async function storeProfilePicture(buffer: Buffer, userId: string) {
}
async function storeCompanyLogo(buffer: Buffer, adjustedEmail: string) {
let quality = 50;
const quality = 50;
const output = await sharp(buffer)
.resize({ width: 300 })
.avif({ quality }) // Verwende AVIF