aknuth/bizmatch-project
1
0
Fork 0
Code Issues 9 Pull Requests Packages Projects 5 Releases Wiki Activity

Umstellung postgres 2. part

Browse Source
This commit is contained in:
Andreas Knuth
2024-04-15 22:05:20 +02:00
parent 7d10080069
commit c4cdcf4505
17 changed files with 18327 additions and 1957 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
crawler/.vscode/launch.json vendored
View File

@@ -16,18 +16,6 @@
"${workspaceFolder}/**/*.js"
]
},
{
"type": "node",
"request": "launch",
"name": "Import",
"skipFiles": [
"<node_internals>/**"
],
"program": "${workspaceFolder}/import.js",
"outFiles": [
"${workspaceFolder}/**/*.js"
]
},
{
"type": "node",
"request": "launch",
@@ -44,11 +32,11 @@
{
"type": "node",
"request": "launch",
"name": "updateFields",
"name": "postgres_business_import",
"skipFiles": [
"<node_internals>/**"
],
"program": "${workspaceFolder}/updateFields.js",
"program": "${workspaceFolder}/build/crawler/postgres_business_import.js",
"outFiles": [
"${workspaceFolder}/**/*.js"
]

3290
crawler/data/businesses.json
View File

File diff suppressed because it is too large Load Diff

12494
crawler/data/businesses_.json
View File

File diff suppressed because it is too large Load Diff

3988
crawler/data/commercials.json
View File

File diff suppressed because it is too large Load Diff

153
crawler/postgres_business_import.ts
View File

@@ -3,6 +3,7 @@ const { Pool } = pkg;
import fsextra from 'fs-extra';
const { fstat, readFileSync, writeJsonSync } = fsextra;
import { v4 as uuidv4 } from 'uuid';
import { CommercialPropertyListing, User } from '../common-models/src/main.model';
// PostgreSQL Verbindungskonfiguration
const pool = new Pool({
user: 'bizmatch',
@@ -39,47 +40,119 @@ interface BusinessListing {
created: Date;
}
// Funktion zum Einlesen und Importieren von JSON-Daten
async function importJsonData(filePath: string): Promise<void> {
try {
const data: string = readFileSync(filePath, 'utf8');
const jsonData: BusinessListing[] = JSON.parse(data); // Erwartet ein Array von Objekten
const out: BusinessListing[] =[]
// Daten für jedes Listing in die Datenbank einfügen
for (const listing of jsonData) {
// const uuid = uuidv4();
// listing.id=uuid;
const values = [
listing.userId, listing.listingsCategory, listing.title, listing.description,
listing.type, listing.state, listing.city, listing.id, listing.price, listing.salesRevenue,
listing.leasedLocation, listing.established, listing.employees,
listing.reasonForSale, listing.supportAndTraining, listing.cashFlow, listing.brokerLicencing,
listing.internalListingNumber, listing.realEstateIncluded, listing.franchiseResale,
listing.draft, listing.internals, listing.created, new Date(), 0, null
];
const json_values = [
listing.id, listing
]
await pool.query(`INSERT INTO businesses
(user_id, listings_category, title, description, type, state, city, id, price, sales_revenue, leased_location,
established, employees, reason_for_sale, support_and_training, cash_flow, broker_licencing, internal_listing_number,
real_estate_included, franchise_resale, draft, internals, created, updated, visits, last_visit)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)`, values);
await pool.query('INSERT INTO businesses_json (id, data) VALUES ($1,$2)', json_values);
// out.push(listing);
}
writeJsonSync('./data/businesses_.json',out);
console.log('All data imported successfully.');
} catch (err) {
console.error('Error importing data:', err.message);
} finally {
// Schließen der Verbindung zum Pool
await pool.end();
async function importBusinesses() {
const filePath = './data/businesses.json'
const data: string = readFileSync(filePath, 'utf8');
const jsonData: BusinessListing[]|any = JSON.parse(data); // Erwartet ein Array von Objekten
await pool.query('drop table if exists businesses');
await pool.query(`CREATE TABLE businesses (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
created TIMESTAMP,
updated TIMESTAMP,
visits INTEGER,
last_visit TIMESTAMP,
data jsonb
);`);
for (const listing of jsonData) {
const created = listing.created
delete listing.created;
delete listing.id;
delete listing.temporary
const json_values = [
created, new Date(), 0, null, listing
]
await pool.query('INSERT INTO businesses (created, updated, visits, last_visit, data) VALUES ($1,$2,$3,$4,$5)', json_values);
}
console.log('All data imported successfully.');
}
async function importUser() {
const filePath = './data/broker.json'
const data: string = readFileSync(filePath, 'utf8');
const jsonData: User[] = JSON.parse(data); // Erwartet ein Array von Objekten
await pool.query('drop table if exists users');
await pool.query(`CREATE TABLE users (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
created TIMESTAMP,
updated TIMESTAMP,
visits INTEGER,
last_visit TIMESTAMP,
data jsonb
);`);
for (const user of jsonData) {
delete user.id;
user.hasCompanyLogo=false;
user.hasProfile=false;
const json_values = [
getRandomDateLastYear(), new Date(), 0, null, user
]
await pool.query('INSERT INTO users (created, updated, visits, last_visit, data) VALUES ($1,$2,$3,$4,$5)', json_values);
}
console.log('All data imported successfully.');
}
async function importCommercials() {
const filePath = './data/commercials.json'
const data: string = readFileSync(filePath, 'utf8');
const jsonData: CommercialPropertyListing[]|any = JSON.parse(data); // Erwartet ein Array von Objekten
await pool.query('drop table if exists commercials');
await pool.query(`CREATE TABLE commercials (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
created TIMESTAMP,
updated TIMESTAMP,
visits INTEGER,
last_visit TIMESTAMP,
data jsonb
);`);
for (const commercial of jsonData) {
commercial.hasImages=false;
commercial.imagePath=commercial.id;
delete commercial.id;
delete commercial.temporary;
const json_values = [
getRandomDateLastYear(), new Date(), 0, null, commercial
]
await pool.query('INSERT INTO commercials (created, updated, visits, last_visit, data) VALUES ($1,$2,$3,$4,$5)', json_values);
}
console.log('All data imported successfully.');
}
function idUpdate(jsonData) {
const out: BusinessListing[] = []
for (const listing of jsonData) {
const uuid = uuidv4();
listing.id = uuid;
out.push(listing);
}
writeJsonSync('./data/businesses_.json', out);
console.log('All data updated sucessfully.');
}
function getRandomDateLastYear(): Date {
const today = new Date();
const lastYear = new Date(today.getFullYear() - 1, today.getMonth(), today.getDate());
// Generiere eine zufällige Zahl zwischen 0 und der Anzahl der Millisekunden in einem Jahr
const randomTime = Math.random() * (today.getTime() - lastYear.getTime());
// Erstelle ein neues Datum basierend auf dieser zufälligen Zeit
const randomDate = new Date(lastYear.getTime() + randomTime);
return randomDate;
}
// Passen Sie den Dateipfad an Ihre spezifischen Bedürfnisse an
importJsonData('./data/businesses_.json');
try {
await importBusinesses();
await importUser();
await importCommercials();
} catch (err) {
console.error('Error importing data:', err.message);
} finally {
// Schließen der Verbindung zum Pool
await pool.end();
}

2
crawler/tsconfig.json
View File

@@ -11,7 +11,7 @@
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "ES2017", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
"target": "ES2020", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */