aknuth/bizmatch-project
1
0
Fork 0
Code Issues 9 Pull Requests Packages Projects 5 Releases Wiki Activity

First version AI Search

Browse Source
This commit is contained in:
Andreas Knuth
2024-07-15 19:37:35 +02:00
parent b7b34dacab
commit af982d19d8
21 changed files with 702 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
bizmatch-server/.vscode/launch.json vendored
View File

@@ -19,13 +19,15 @@
{
"type": "node",
"request": "launch",
"name": "Debug Current TS File",
"program": "${workspaceFolder}/dist/src/drizzle/${fileBasenameNoExtension}.js",
"preLaunchTask": "tsc: build - tsconfig.json",
"outFiles": ["${workspaceFolder}/out/**/*.js"],
"name": "Launch TypeScript file with tsx",
"runtimeExecutable": "npx",
"runtimeArgs": ["tsx", "--inspect"],
"args": ["${workspaceFolder}/src/drizzle/import.ts"],
"cwd": "${workspaceFolder}",
"outFiles": ["${workspaceFolder}/dist/**/*.js", "!**/node_modules/**"],
"sourceMaps": true,
"smartStep": true,
"internalConsoleOptions": "openOnSessionStart"
"resolveSourceMapLocations": ["${workspaceFolder}/src/**/*.ts", "!**/node_modules/**"],
"skipFiles": ["<node_internals>/**", "${workspaceFolder}/node_modules/**/*.js"]
},
{
"type": "node",

5
bizmatch-server/package.json
View File

@@ -54,7 +54,7 @@
"reflect-metadata": "^0.2.0",
"rxjs": "^7.8.1",
"sharp": "^0.33.2",
"tsx": "^4.7.2",
"tsx": "^4.16.2",
"urlcat": "^3.1.0",
"winston": "^3.11.0"
},
@@ -78,6 +78,7 @@
"@typescript-eslint/parser": "^6.0.0",
"commander": "^12.0.0",
"drizzle-kit": "^0.23.0",
"esbuild-register": "^3.5.0",
"eslint": "^8.42.0",
"eslint-config-prettier": "^9.0.0",
"eslint-plugin-prettier": "^5.0.0",
@@ -111,4 +112,4 @@
"coverageDirectory": "../coverage",
"testEnvironment": "node"
}
}
}

112
bizmatch-server/src/drizzle/import.ts
View File

@@ -7,10 +7,12 @@ import { join } from 'path';
import pkg from 'pg';
import { rimraf } from 'rimraf';
import sharp from 'sharp';
import { SelectOptionsService } from 'src/select-options/select-options.service.js';
import winston from 'winston';
import { BusinessListing, CommercialPropertyListing, User, UserData } from '../models/db.model.js';
import { emailToDirName, KeyValueStyle } from '../models/main.model.js';
import * as schema from './schema.js';
import { users } from './schema.js';
const typesOfBusiness: Array<KeyValueStyle> = [
{ name: 'Automotive', value: '1', icon: 'fa-solid fa-car', textColorClass: 'text-green-400' },
{ name: 'Industrial Services', value: '2', icon: 'fa-solid fa-industry', textColorClass: 'text-yellow-400' },
@@ -44,6 +46,7 @@ await db.delete(schema.commercials);
await db.delete(schema.businesses);
await db.delete(schema.users);
const sso = new SelectOptionsService();
//Broker
let filePath = `./data/broker.json`;
let data: string = readFileSync(filePath, 'utf8');
@@ -63,6 +66,8 @@ fs.ensureDirSync(`./pictures/logo`);
fs.ensureDirSync(`./pictures/profile`);
fs.ensureDirSync(`./pictures/property`);
type UserProfile = Omit<User, 'created' | 'updated' | 'hasCompanyLogo' | 'hasProfile' | 'id'>;
type NewUser = typeof users.$inferInsert;
//for (const userData of usersData) {
for (let index = 0; index < usersData.length; index++) {
const userData = usersData[index];
@@ -100,18 +105,17 @@ for (let index = 0; index < usersData.length; index++) {
const userProfile = createUserProfile(user);
logger.info(`${index} - ${JSON.stringify(userProfile)}`);
const embedding = await createEmbedding(JSON.stringify(userProfile));
sleep(500);
sleep(200);
const u = await db
.insert(schema.users)
.values({
...user,
embedding: embedding,
})
.returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email });
// const u = await db.insert(schema.users).values(user).returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email });
} as NewUser)
.returning({ insertedId: schema.users.id, gender: schema.users.gender, email: schema.users.email, firstname: schema.users.firstname, lastname: schema.users.lastname });
generatedUserData.push(u[0]);
i++;
logger.info(`user_${index} inserted`);
if (u[0].gender === 'male') {
male++;
const data = readFileSync(`./pictures_base/profile/Mann_${male}.jpg`);
@@ -125,46 +129,12 @@ for (let index = 0; index < usersData.length; index++) {
await storeCompanyLogo(data, emailToDirName(u[0].email));
}
//Business Listings
filePath = `./data/businesses.json`;
data = readFileSync(filePath, 'utf8');
const businessJsonData = JSON.parse(data) as BusinessListing[]; // Erwartet ein Array von Objekten
for (const business of businessJsonData) {
delete business.id;
business.created = new Date(business.created);
business.updated = new Date(business.created);
const user = getRandomItem(generatedUserData);
business.email = user.email;
business.imageName = emailToDirName(user.email);
const embeddingText = JSON.stringify({
type: typesOfBusiness.find(b => b.value === String(business.type))?.name,
title: business.title,
description: business.description,
city: business.city,
state: business.state,
price: business.price,
realEstateIncluded: business.realEstateIncluded,
leasedLocation: business.leasedLocation,
franchiseResale: business.franchiseResale,
salesRevenue: business.salesRevenue,
cashFlow: business.cashFlow,
supportAndTraining: business.supportAndTraining,
employees: business.employees,
established: business.established,
reasonForSale: business.reasonForSale,
});
const embedding = await createEmbedding(embeddingText);
sleep(300);
await db.insert(schema.businesses).values({
...business,
embedding: embedding,
});
}
//Corporate Listings
filePath = `./data/commercials.json`;
data = readFileSync(filePath, 'utf8');
const commercialJsonData = JSON.parse(data) as CommercialPropertyListing[]; // Erwartet ein Array von Objekten
for (const commercial of commercialJsonData) {
for (let index = 0; index < commercialJsonData.length; index++) {
const commercial = commercialJsonData[index];
const id = commercial.id;
delete commercial.id;
const user = getRandomItem(generatedUserData);
@@ -175,7 +145,25 @@ for (const commercial of commercialJsonData) {
commercial.updated = insertionDate;
commercial.email = user.email;
commercial.draft = false;
const result = await db.insert(schema.commercials).values(commercial).returning();
const reducedCommercial = {
city: commercial.city,
description: commercial.description,
email: commercial.email,
price: commercial.price,
state: sso.locations.find(l => l.value === commercial.state)?.name,
title: commercial.title,
name: `${user.firstname} ${user.lastname}`,
};
const embedding = await createEmbedding(JSON.stringify(reducedCommercial));
sleep(200);
const result = await db
.insert(schema.commercials)
.values({
...commercial,
embedding: embedding,
})
.returning();
logger.info(`commercial_${index} inserted`);
try {
fs.copySync(`./pictures_base/property/${id}`, `./pictures/property/${result[0].imagePath}/${result[0].serialId}`);
} catch (err) {
@@ -183,6 +171,46 @@ for (const commercial of commercialJsonData) {
}
}
//Business Listings
filePath = `./data/businesses.json`;
data = readFileSync(filePath, 'utf8');
const businessJsonData = JSON.parse(data) as BusinessListing[]; // Erwartet ein Array von Objekten
for (let index = 0; index < businessJsonData.length; index++) {
const business = businessJsonData[index];
delete business.id;
business.created = new Date(business.created);
business.updated = new Date(business.created);
const user = getRandomItem(generatedUserData);
business.email = user.email;
business.imageName = emailToDirName(user.email);
const embeddingText = JSON.stringify({
type: typesOfBusiness.find(b => b.value === String(business.type))?.name,
title: business.title,
description: business.description,
email: business.email,
city: business.city,
state: sso.locations.find(l => l.value === business.state)?.name,
price: business.price,
realEstateIncluded: business.realEstateIncluded,
leasedLocation: business.leasedLocation,
franchiseResale: business.franchiseResale,
salesRevenue: business.salesRevenue,
cashFlow: business.cashFlow,
supportAndTraining: business.supportAndTraining,
employees: business.employees,
established: business.established,
reasonForSale: business.reasonForSale,
name: `${user.firstname} ${user.lastname}`,
});
const embedding = await createEmbedding(embeddingText);
sleep(200);
await db.insert(schema.businesses).values({
...business,
embedding: embedding,
});
logger.info(`business_${index} inserted`);
}
//End
await client.end();

8
bizmatch-server/src/listings/business-listings.controller.ts
View File

@@ -26,11 +26,17 @@ export class BusinessListingsController {
}
@UseGuards(OptionalJwtAuthGuard)
@Post('search')
@Post('find')
find(@Request() req, @Body() criteria: ListingCriteria): any {
return this.listingsService.findBusinessListings(criteria, req.user as JwtUser);
}
@UseGuards(OptionalJwtAuthGuard)
@Post('search')
search(@Request() req, @Body() criteria: ListingCriteria): any {
return this.listingsService.searchBusinessListings(criteria.prompt);
}
@Post()
create(@Body() listing: any) {
this.logger.info(`Save Listing`);

2
bizmatch-server/src/listings/commercial-property-listings.controller.ts
View File

@@ -28,7 +28,7 @@ export class CommercialPropertyListingsController {
return this.listingsService.findCommercialPropertiesByEmail(email, req.user as JwtUser);
}
@UseGuards(OptionalJwtAuthGuard)
@Post('search')
@Post('find')
async find(@Request() req, @Body() criteria: ListingCriteria): Promise<any> {
return await this.listingsService.findCommercialPropertyListings(criteria, req.user as JwtUser);
}

61
bizmatch-server/src/listings/listings.service.ts
View File

@@ -2,6 +2,7 @@ import { Inject, Injectable } from '@nestjs/common';
import { and, eq, gte, ilike, lte, ne, or, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { WINSTON_MODULE_PROVIDER } from 'nest-winston';
import OpenAI from 'openai';
import { Logger } from 'winston';
import * as schema from '../drizzle/schema.js';
import { PG_CONNECTION, businesses, commercials } from '../drizzle/schema.js';
@@ -11,11 +12,16 @@ import { JwtUser, ListingCriteria, emailToDirName } from '../models/main.model.j
@Injectable()
export class ListingsService {
openai: OpenAI;
constructor(
@Inject(WINSTON_MODULE_PROVIDER) private readonly logger: Logger,
@Inject(PG_CONNECTION) private conn: NodePgDatabase<typeof schema>,
private fileService: FileService,
) {}
) {
this.openai = new OpenAI({
apiKey: process.env.OPENAI_API_KEY, // Stellen Sie sicher, dass Sie Ihren API-Key als Umgebungsvariable setzen
});
}
private getConditions(criteria: ListingCriteria, table: typeof businesses | typeof commercials, user: JwtUser): any[] {
const conditions = [];
if (criteria.type) {
@@ -42,6 +48,19 @@ export class ListingsService {
// Listings general
// ##############################################################
// #### Find by embeddng ########################################
async searchBusinessListings(query: string, limit: number = 20): Promise<BusinessListing[]> {
const queryEmbedding = await this.createEmbedding(query);
const results = await this.conn
.select()
.from(businesses)
.orderBy(sql`embedding <-> ${JSON.stringify(queryEmbedding)}`)
.limit(limit);
return results as BusinessListing[];
}
// #### Find by criteria ########################################
async findCommercialPropertyListings(criteria: ListingCriteria, user: JwtUser): Promise<any> {
const start = criteria.start ? criteria.start : 0;
const length = criteria.length ? criteria.length : 12;
@@ -86,6 +105,8 @@ export class ListingsService {
]);
return { total, data };
}
// #### Find by ID ########################################
async findCommercialPropertiesById(id: string, user: JwtUser): Promise<CommercialPropertyListing> {
let result = await this.conn
.select()
@@ -102,6 +123,8 @@ export class ListingsService {
result = result.filter(r => !r.draft || r.imageName === emailToDirName(user?.username) || user?.roles.includes('ADMIN'));
return result[0] as BusinessListing;
}
// #### Find by User EMail ########################################
async findCommercialPropertiesByEmail(email: string, user: JwtUser): Promise<CommercialPropertyListing[]> {
const conditions = [];
conditions.push(eq(commercials.imagePath, emailToDirName(email)));
@@ -124,6 +147,8 @@ export class ListingsService {
.from(businesses)
.where(and(...conditions))) as CommercialPropertyListing[];
}
// #### Find by imagePath ########################################
async findByImagePath(imagePath: string, serial: string): Promise<CommercialPropertyListing> {
const result = await this.conn
.select()
@@ -131,13 +156,15 @@ export class ListingsService {
.where(and(sql`${commercials.imagePath} = ${imagePath}`, sql`${commercials.serialId} = ${serial}`));
return result[0] as CommercialPropertyListing;
}
// #### CREATE ########################################
async createListing(data: BusinessListing | CommercialPropertyListing, table: typeof businesses | typeof commercials): Promise<BusinessListing | CommercialPropertyListing> {
data.created = new Date();
data.updated = new Date();
const [createdListing] = await this.conn.insert(table).values(data).returning();
return createdListing as BusinessListing | CommercialPropertyListing;
}
// #### UPDATE CommercialProps ########################################
async updateCommercialPropertyListing(id: string, data: CommercialPropertyListing): Promise<BusinessListing | CommercialPropertyListing> {
data.updated = new Date();
data.created = new Date(data.created);
@@ -150,22 +177,18 @@ export class ListingsService {
const [updateListing] = await this.conn.update(commercials).set(data).where(eq(commercials.id, id)).returning();
return updateListing as BusinessListing | CommercialPropertyListing;
}
// #### UPDATE Business ########################################
async updateBusinessListing(id: string, data: BusinessListing): Promise<BusinessListing | CommercialPropertyListing> {
data.updated = new Date();
data.created = new Date(data.created);
const [updateListing] = await this.conn.update(businesses).set(data).where(eq(businesses.id, id)).returning();
return updateListing as BusinessListing | CommercialPropertyListing;
}
// #### DELETE ########################################
async deleteListing(id: string, table: typeof businesses | typeof commercials): Promise<void> {
await this.conn.delete(table).where(eq(table.id, id));
}
async getStates(table: typeof businesses | typeof commercials): Promise<any[]> {
return await this.conn
.select({ state: table.state, count: sql<number>`count(${table.id})`.mapWith(Number) })
.from(table)
.groupBy(sql`${table.state}`)
.orderBy(sql`count desc`);
}
// ##############################################################
// Images for commercial Properties
// ##############################################################
@@ -182,4 +205,24 @@ export class ListingsService {
listing.imageOrder.push(imagename);
await this.updateCommercialPropertyListing(listing.id, listing);
}
// ##############################################################
// States
// ##############################################################
async getStates(table: typeof businesses | typeof commercials): Promise<any[]> {
return await this.conn
.select({ state: table.state, count: sql<number>`count(${table.id})`.mapWith(Number) })
.from(table)
.groupBy(sql`${table.state}`)
.orderBy(sql`count desc`);
}
// ##############################################################
// Embedding
// ##############################################################
async createEmbedding(text: string): Promise<number[]> {
const response = await this.openai.embeddings.create({
model: 'text-embedding-3-small',
input: text,
});
return response.data[0].embedding;
}
}

1
bizmatch-server/src/models/main.model.ts
View File

@@ -65,6 +65,7 @@ export interface ListingCriteria {
title: string;
category: 'professional' | 'broker';
name: string;
prompt: string;
}
export interface KeycloakUser {