donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
176 lines • 8.42 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ExtractGoogleStreetviewEntityDataTool = exports.PropertyType = void 0;
const Tool_1 = require("./Tool");
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
const MiscUtils_1 = require("../utils/MiscUtils");
const NavigateWithinStreetView_1 = require("./NavigateWithinStreetView");
const AggregateExtractedStreetviewDataTool_1 = require("./AggregateExtractedStreetviewDataTool");
var PropertyType;
(function (PropertyType) {
PropertyType["RESIDENTIAL"] = "RESIDENTIAL";
PropertyType["RETAIL"] = "RETAIL";
PropertyType["OFFICE_BUILDING"] = "OFFICE_BUILDING";
PropertyType["INDUSTRIAL"] = "INDUSTRIAL";
PropertyType["EMPTY_LOT"] = "EMPTY_LOT";
})(PropertyType || (exports.PropertyType = PropertyType = {}));
/**
* Extracts data from the current Google Maps Street View. If the current page is not a Google Maps
* Street View, then the operation is undefined.
*/
class ExtractGoogleStreetviewEntityDataTool extends Tool_1.Tool {
constructor() {
super(ExtractGoogleStreetviewEntityDataTool.NAME, `Extract business/entity data from the current Google Maps street view, paying particular attention for a given target
business/entity. This tool should be called for each unique street view.`, 'ExtractGoogleStreetviewEntityDataToolCoreParameters', 'ExtractGoogleStreetviewEntityDataToolGptParameters', true);
}
async call(context, parameters) {
const page = context.page;
const firstCallForThisUrl = context.invokedToolCalls.find((tc) => tc.toolName === this.name && tc.page === page.url());
// Remove Google Maps survey if present
await NavigateWithinStreetView_1.NavigateWithinStreetViewTool.removeGoogleMapsSurvey(page);
if (firstCallForThisUrl) {
await context.toolTipper.blipToolTip(page, `Skipping ${this.name} call since it has already extracted data for this view.`);
const originalRec = firstCallForThisUrl.parameters
.recommendedNextStepForStreetViewNavigator;
const advice = !originalRec
? ''
: `The prior recommendedNextStepForStreetViewNavigator was \`${originalRec}\`, maybe try something else.`;
return {
isSuccessful: true,
forLlm: `Skipping ${this.name} call since it has already extracted data for this view.
Call the ${NavigateWithinStreetView_1.NavigateWithinStreetViewTool.NAME} tool next!
${advice}`.trim(),
metadata: null,
};
}
else {
const entityName = parameters.entityName.trim();
await this.wiggleMouseThenPause(page);
const screenshot = await PlaywrightUtils_1.PlaywrightUtils.takePngScreenshot(page);
const screenshotFilename = await context.persistence.savePngScreenShot(context.metadata.id, screenshot);
const googleMapsStreetviewUrl = page.url();
const entityData = await this.extractEntityData(context, screenshot);
const filteredBusinessSignage = entityData.businessSignage.filter((sign) => sign && sign.trim() !== '');
const otherSignage = entityData.otherSignage
? entityData.otherSignage.filter((sign) => sign && sign.trim() !== '')
: [];
const doesBusinessExistInSignage = await this.doesBusinessExistInSignage(context, entityName, filteredBusinessSignage);
const metadata = {
businessSignage: filteredBusinessSignage,
otherSignage,
isBusinessMatch: doesBusinessExistInSignage,
imageId: screenshotFilename,
googleMapsStreetviewUrl,
typeOfProperty: entityData.typeOfProperty,
recommendedNextStepForStreetViewNavigator: entityData.recommendedNextStepForStreetViewNavigator,
};
if (doesBusinessExistInSignage) {
context.proposedToolCalls.push({
name: AggregateExtractedStreetviewDataTool_1.AggregateExtractedStreetviewDataTool.NAME,
parameters: {},
});
}
const dataForGpt = JSON.stringify(metadata, null, 2);
return { isSuccessful: true, forLlm: dataForGpt, metadata: metadata };
}
}
callFromGpt(context, parameters) {
return this.call(context, parameters);
}
async wiggleMouseThenPause(page) {
const rand = Math.random();
const x = 650 + Math.floor(rand * 10);
const y = 550 + Math.floor(rand * 10);
const wiggleDistance = 5 + Math.floor(rand * 5);
await page.mouse.move(x, y);
await page.waitForTimeout(10 + Math.floor(rand * 50));
await page.mouse.move(x + wiggleDistance, y);
await page.waitForTimeout(10 + Math.floor(rand * 50));
await page.mouse.move(x, y + wiggleDistance);
await page.waitForTimeout(10 + Math.floor(rand * 50));
await page.mouse.move(x - wiggleDistance, y);
await page.waitForTimeout(10 + Math.floor(rand * 50));
await page.mouse.move(x, y - wiggleDistance);
await page.waitForTimeout(5000);
}
async extractEntityData(context, screenshot) {
const promptText = {
type: 'text',
text: 'You are a screenshot from google maps street view navigator. Extract the business data in this Google Maps street view and recommend next steps for street view navigator to view the building better.',
};
const userMessage = {
type: 'user',
items: [{ type: 'png', bytes: screenshot }, promptText],
};
const gptResp = await context.gptClient.getStructuredOutput([userMessage], ExtractGoogleStreetviewEntityDataTool.EXTRACT_BUSINESS_DATA_SCHEMA);
MiscUtils_1.MiscUtils.updateTokenCounts(gptResp, context.metadata);
return gptResp.output;
}
async doesBusinessExistInSignage(context, businessName, businessSignage) {
const promptText = {
type: 'text',
text: `Does the business/entity, ${businessName}, exist in this list of business signage? Signage: ${businessSignage}`,
};
const userMessage = {
type: 'user',
items: [promptText],
};
const gptResp = await context.gptClient.getStructuredOutput([userMessage], ExtractGoogleStreetviewEntityDataTool.DOES_BUSINESS_EXIST_SCHEMA);
MiscUtils_1.MiscUtils.updateTokenCounts(gptResp, context.metadata);
return gptResp.output.isBusinessMatch;
}
}
exports.ExtractGoogleStreetviewEntityDataTool = ExtractGoogleStreetviewEntityDataTool;
ExtractGoogleStreetviewEntityDataTool.NAME = 'extractGoogleStreetviewEntityData';
ExtractGoogleStreetviewEntityDataTool.EXTRACT_BUSINESS_DATA_SCHEMA = {
type: 'object',
required: [
'businessSignage',
'otherSignage',
'typeOfProperty',
'recommendedNextStepForStreetViewNavigator',
],
properties: {
businessSignage: {
type: 'array',
description: 'A list of all of the business signage text.',
items: {
type: 'string',
},
},
otherSignage: {
type: 'array',
description: 'A list of all of the non-business signage text.',
items: {
type: 'string',
},
},
typeOfProperty: {
type: 'string',
enum: [
'RESIDENTIAL',
'RETAIL',
'OFFICE_BUILDING',
'INDUSTRIAL',
'EMPTY_LOT',
],
},
recommendedNextStepForStreetViewNavigator: {
type: 'string',
enum: ['TURN_LEFT', 'TURN_RIGHT', 'GO_FORWARD', 'TURN_AROUND'], // assuming these are the Action enum values
},
},
additionalProperties: false,
};
ExtractGoogleStreetviewEntityDataTool.DOES_BUSINESS_EXIST_SCHEMA = {
type: 'object',
required: ['isBusinessMatch'],
properties: {
isBusinessMatch: {
type: 'boolean',
description: 'Set to true IF AND ONLY IF the business being sought exists in given business signage.',
},
},
additionalProperties: false,
};
//# sourceMappingURL=ExtractGoogleStreetviewEntityDataTool.js.map