commit
This commit is contained in:
parent
be4fd23bcf
commit
0bd53741af
728 changed files with 86573 additions and 0 deletions
101
node_modules/googlethis/lib/utils/constants.js
generated
vendored
Normal file
101
node_modules/googlethis/lib/utils/constants.js
generated
vendored
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
'use strict';
|
||||
|
||||
module.exports = {
|
||||
URLS: {
|
||||
GIS: 'https://images.google.com/',
|
||||
GOOGLE: 'https://google.com/',
|
||||
W_GOOGLE: 'https://www.google.com/',
|
||||
GOOGLE_NEWS: 'https://news.google.com/',
|
||||
FAVICONKIT: 'https://api.faviconkit.com'
|
||||
},
|
||||
SELECTORS: {
|
||||
// Organic Search Results
|
||||
TITLE: 'div.ynAwRc.q8U8x.MBeuO.gsrt.oewGkc.LeUQr',
|
||||
DESCRIPTION: 'div.MUxGbd.yDYNvb',
|
||||
URL: 'a.C8nzq.BmP5tf',
|
||||
|
||||
// Did You Mean
|
||||
DID_YOU_MEAN: 'a.gL9Hy',
|
||||
|
||||
// Knowledge Panel
|
||||
KNO_PANEL_TITLE: [ 'div.BkwXh > div', 'div > span.u9DLmf' ],
|
||||
KNO_PANEL_DESCRIPTION: 'div[class="kno-rdesc"] > span',
|
||||
KNO_PANEL_URL: 'div[class="kno-rdesc"] > span > a',
|
||||
KNO_PANEL_METADATA: 'div.rVusze > span',
|
||||
KNO_PANEL_TYPE: 'div.BkwXh > div',
|
||||
KNO_PANEL_SONG_LYRICS: 'div.ujudUb',
|
||||
KNO_PANEL_AVAILABLE_ON: 'div[class="ellip bclEt"]',
|
||||
KNO_PANEL_IMAGES: 'g-inner-card > div > div > img',
|
||||
KNO_PANEL_SONGS: 'a > div > div > div > div[class="title"]',
|
||||
KNO_PANEL_BOOKS: 'div[data-attrid="kc:/book/author:books only"] > a > div > div > div.Bo9xMe > div',
|
||||
KNO_PANEL_TV_SHOWS_AND_MOVIES: 'div[data-attrid="kc:/people/person:tv-shows-and-movies"] > a > div > div > div.Bo9xMe > div',
|
||||
KNO_PANEL_FILM_GOOGLEUSERS_RATING: 'div[data-attrid="kc:/ugc:thumbs_up"] > div > div > div',
|
||||
KNO_PANEL_FILM_RATINGS: ['span[class="gsrt KMdzJ"]', 'span[class="rhsB pVA7K"]'],
|
||||
KNO_PANEL_SOCIALS: 'div[data-attrid="kc:/common/topic:social media presence"] > div > kp-carousel > g-scrolling-carousel > div > div > kp-carousel-item > div > g-link > a',
|
||||
|
||||
VIDEOS: 'div > div > div > div > video-voyager > div',
|
||||
|
||||
// Featured Snippet
|
||||
FEATURED_SNIPPET_TITLE: ['div[class="co8aDb gsrt"]', 'a[class="sXtWJb gsrt"]', 'div[class="Xv4xee"]'],
|
||||
FEATURED_SNIPPET_DESC: ['ol[class="X5LH0c"]', 'ul[class="i8Z77e"]', 'div[data-attrid="wa:/description"]'],
|
||||
FEATURED_SNIPPET_URL: 'div > div > h3 > a',
|
||||
|
||||
// Unit converter
|
||||
UNIT_CONVERTER_INPUT: 'div.rpnBye > input',
|
||||
UNIT_CONVERTER_OUTPUT: 'div[id="NotFQb"] > input',
|
||||
UNIT_CONVERTER_FORMULA: 'div.bjhkR',
|
||||
INPUT_CURRENCY_NAME: 'span.vLqKYe',
|
||||
OUTPUT_CURRENCY_NAME: 'span.MWvIVe',
|
||||
CURRENCY_CONVERTER_INPUT: 'span.DFlfde.eNFL1',
|
||||
CURRENCY_CONVERTER_OUTPUT: 'span.DFlfde.SwHCTb',
|
||||
|
||||
// Weather forecast
|
||||
WEATHER_LOCATION: 'div.wob_hdr > div[id="wob_loc"]',
|
||||
WEATHER_FORECAST: 'div.wob_dsc',
|
||||
PRECIPITATION: 'div.wob_dtf > div > span[id="wob_pp"]',
|
||||
AIR_HUMIDITY: 'div.wob_dtf > div > span[id="wob_hm"]',
|
||||
TEMPERATURE: 'div > span[id="wob_tm"]',
|
||||
WIND_SPEED: 'span[id="wob_ws"]',
|
||||
|
||||
// Time result, E.g: try searching “what time is it in Japan?”
|
||||
CURRENT_TIME_HOUR: 'div > div[role="heading"]',
|
||||
CURRENT_TIME_DATE: 'div.vk_gy.vk_sh',
|
||||
|
||||
// Location result
|
||||
LOCATION_TITLE: 'div.vk_sh.vk_gy',
|
||||
LOCATION_DISTANCE: 'div.dDoNo.FzvWSb.vk_bk',
|
||||
LOCATION_IMAGE: 'div.vk_c > div > a > img',
|
||||
|
||||
// Google Dictionary
|
||||
GD_WORD: 'span[data-dobid="hdw"]',
|
||||
GD_PHONETIC: 'div.qexShd',
|
||||
GD_AUDIO: 'audio > source',
|
||||
GD_DEFINITIONS: 'div[data-dobid="dfn"]',
|
||||
GD_EXAMPLES: 'div[class="ubHt5c"]',
|
||||
|
||||
// Google Translator
|
||||
TR_SOURCE_LANGUAGE: 'div[class="j1iyq"] > span[class="source-language"]',
|
||||
TR_TARGET_LANGUAGE: 'div[class="j1iyq"] > span[class="target-language"]',
|
||||
|
||||
TR_SOURCE_TEXT: 'pre[id="tw-source-text"] > span[class="Y2IQFc"]',
|
||||
TR_TARGET_TEXT: 'pre[id="tw-target-text"] > span[class="Y2IQFc"]',
|
||||
|
||||
// Top Stories
|
||||
TOP_STORIES_DESCRIPTION: ['div.g5wfEd', 'div.VeOk3'],
|
||||
TOP_STORIES_URL: 'a.WlydOe.amp_r',
|
||||
TOP_STORIES_SNIPPET: 'div[class="g5wfEd"] > div[role="heading"]',
|
||||
TOP_STORIES_WEBSITE: 'div[class="g5wfEd"] > div > g-img > img',
|
||||
|
||||
// “People also ask”
|
||||
PAA: [ 'div.s75CSd.u60jwe.gduDCb > span', 'div.gbCQS.u60jwe.gduDCb > div > span', 'div.JlqpRe > span' ],
|
||||
|
||||
// “People also search for”
|
||||
PASF: 'div[class="IHdOHf"] > img',
|
||||
|
||||
// Top News
|
||||
PUBLISHER: 'a[data-n-tid="9"]',
|
||||
STORY_TITLE: 'a[class="DY5T1d RZIKme"]',
|
||||
STORY_IMG: 'img[class="tvs3Id QwxBBf"]',
|
||||
STORY_TIME: 'time'
|
||||
}
|
||||
};
|
||||
42
node_modules/googlethis/lib/utils/user-agents.json
generated
vendored
Normal file
42
node_modules/googlethis/lib/utils/user-agents.json
generated
vendored
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
{
|
||||
"desktop": [
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36 Edg/103.0.1264.62",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36 Edg/103.0.1264.49",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36"
|
||||
],
|
||||
"mobile": [
|
||||
"Mozilla/5.0 (Linux; Android 12; SAMSUNG SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 11; SM-G781B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 12; SM-G973F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 11; Redmi Note 8 Pro) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 11; ONEPLUS A6013) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
|
||||
"Mozilla/5.0 (Linux; Android 12; SM-G986B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/103.0.5060.63 Mobile/15E148 Safari/604.1",
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Mobile/15E148 Safari/604.1",
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/103.0.5060.63 Mobile/15E148 Safari/604.1",
|
||||
"Mozilla/5.0 (iPhone; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.25 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"
|
||||
]
|
||||
}
|
||||
131
node_modules/googlethis/lib/utils/utils.js
generated
vendored
Normal file
131
node_modules/googlethis/lib/utils/utils.js
generated
vendored
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
'use strict';
|
||||
|
||||
const userAgents = require('./user-agents.json');
|
||||
|
||||
class SearchError extends Error {
|
||||
constructor (message, info) {
|
||||
super(message);
|
||||
|
||||
info && (this.info = info);
|
||||
|
||||
this.date = new Date();
|
||||
this.version = require('../../package.json').version;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns headers with a random user agent.
|
||||
*
|
||||
* @param {boolean} is_mobile
|
||||
* @returns {string}
|
||||
*/
|
||||
function getHeaders(options = { mobile: false }) {
|
||||
const available_agents = userAgents[options.mobile ? 'mobile' : 'desktop'];
|
||||
const ua = available_agents[Math.floor(Math.random() * available_agents.length)];
|
||||
|
||||
return {
|
||||
'accept': 'text/html',
|
||||
'accept-encoding': 'gzip, deflate',
|
||||
'accept-language': 'en-US,en',
|
||||
'referer': 'https://www.google.com/',
|
||||
'upgrade-insecure-requests': 1,
|
||||
'user-agent': ua
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Refines the html.
|
||||
*
|
||||
* @param {string} data - Raw html data.
|
||||
* @param {boolean} parse_ads - Whether to parse ads or not.
|
||||
* @returns {string}
|
||||
*/
|
||||
function refineData (data, parse_ads = false, is_mobile = true) {
|
||||
let result = data
|
||||
// Removes classes we don't need:
|
||||
.replace(/N6jJud MUxGbd lyLwlc/g, '')
|
||||
.replace(/YjtGef ExmHv MUxGbd/g, '')
|
||||
.replace(/MUxGbd lyLwlc aLF0Z/g, '')
|
||||
|
||||
/*
|
||||
* Transforms all possible variations of some classes' name into a
|
||||
* fixed string so it's easier to get consistent results:
|
||||
**/
|
||||
|
||||
// Descriptions: -> MUxGbd yDYNvb
|
||||
.replace(/yDYNvb lEBKkf/g, 'yDYNvb')
|
||||
.replace(/VwiC3b MUxGbd yDYNvb/g, 'MUxGbd yDYNvb')
|
||||
|
||||
// Urls: -> C8nzq BmP5tf
|
||||
.replace(/cz3goc BmP5tf/g, 'C8nzq BmP5tf')
|
||||
|
||||
// Titles: -> ynAwRc q8U8x MBeuO gsrt oewGkc LeUQr
|
||||
.replace(/ynAwRc q8U8x MBeuO oewGkc LeUQr/g, 'ynAwRc q8U8x MBeuO gsrt oewGkc LeUQr')
|
||||
.replace(/MBeuO oewGkc/g, 'MBeuO gsrt oewGkc');
|
||||
|
||||
// Transform desktop title/urls classes. Everything else is the same.
|
||||
if (!is_mobile) {
|
||||
result = result
|
||||
.replace(/yuRUbf|v5yQqb/g, 'ynAwRc q8U8x MBeuO gsrt oewGkc LeUQr')
|
||||
}
|
||||
|
||||
// Transform ad title classes.
|
||||
if (parse_ads) {
|
||||
result = result
|
||||
.replace(/cz3goc v5yQqb BmP5tf/g, 'C8nzq BmP5tf')
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a string between two delimiters.
|
||||
*
|
||||
* @param {string} data - The data.
|
||||
* @param {string} start_string - Start string.
|
||||
* @param {string} end_string - End string.
|
||||
*
|
||||
* @returns {string}
|
||||
*/
|
||||
function getStringBetweenStrings (data, start_string, end_string) {
|
||||
const regex = new RegExp(`${escapeStringRegexp(start_string)}(.*?)${escapeStringRegexp(end_string)}`, 's');
|
||||
const match = data.match(regex);
|
||||
return match ? match[1] : undefined;
|
||||
}
|
||||
|
||||
function escapeStringRegexp (string) {
|
||||
return string.replace(/[|\\{}()[\]^$+*?.]/g, '\\$&').replace(/-/g, '\\x2d');
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates a random string with a given length.
|
||||
* @param {number} length
|
||||
* @returns {string}
|
||||
*/
|
||||
function generateRandomString(length) {
|
||||
const result = [];
|
||||
const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_';
|
||||
|
||||
for (let i = 0; i < length; i++) {
|
||||
result.push(alphabet.charAt(Math.floor(Math.random() * alphabet.length)));
|
||||
}
|
||||
|
||||
return result.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a random integer between two values.
|
||||
*
|
||||
* @param {number} min
|
||||
* @param {number} max
|
||||
*
|
||||
* @returns {number}
|
||||
*/
|
||||
function getRandomInt(min, max) {
|
||||
min = Math.ceil(min);
|
||||
max = Math.floor(max);
|
||||
return Math.floor(Math.random() * (max - min) + min); //The maximum is exclusive and the minimum is inclusive
|
||||
}
|
||||
|
||||
module.exports = { SearchError, getHeaders, getStringBetweenStrings, generateRandomString, getRandomInt, refineData };
|
||||
Loading…
Add table
Add a link
Reference in a new issue