Specter is a Node.js wrapper for PhantomJS. PhantomJS is a headless browser which enables developers to load pages that rely on Javascript. Specter was made to easily access these dynamically and asynchronously created HTML pages using Node.js.
This returns the page when it has loaded. Generally this will not return any dynamic context, but it is a good start.
var specter = require('specter');
var urls = [];
urls.push({
address: 'https://www.coindega.com/api/0.0.0/#items'
});
specter.process(urls, function (err, url, html) {
if (err) {
console.log(err);
} else if (!html) {
console.log('nothing found')
} else {
console.log(url);
console.log(html);
}
});
By providing select Specter will wait for until document.querySelect('.your .item') to return a value. Once it returns a value the entire html document will be returned.
var specter = require('specter');
var urls = [];
urls.push({
address: 'https://www.coindega.com/api/0.0.0/#items',
select: '.items .title'
});
specter.process(urls, function (err, url, html) {
if (err) {
console.log(err);
} else if (!html) {
console.log('nothing found')
} else {
console.log(url);
console.log(html);
}
});
This does the same thing
var specter = require('specter');
var urls = [];
urls.push({
address: 'https://www.coindega.com/api/0.0.0/#items',
select: '.items .title',
wait: (10 * 1000) // return after 10 seconds if nothing found
});
specter.process(urls, function (err, html) {
if (err) {
console.log(err);
} else if (!html) {
console.log('nothing found')
} else {
console.log(html);
}
});
Specter has no dependencies so the response is just a string; howerver, it is trivial to add jQuery.
var jquery = require('jquery');
var specter = require('specter');
var urls = [];
urls.push({
address: 'https://www.coindega.com/api/0.0.0/#items',
select: '.items .title',
wait: (10 * 1000) // return after 10 seconds if nothing found
});
specter.process(urls, function (err, url, html) {
if (err) {
console.log(err);
} else if (!html) {
console.log('nothing found')
} else {
var $ = jquery(html);
var $el = $.find('.items .title');
var $first = $el.first();
var text = $first.text();
console.log(url);
console.log(text);
}
});
If you do not need to wait for the Javascript to load Specter can get the page with curl. Obviusly wait and select cannot be used in this scenario.
var specter = require('specter');
var urls = [];
urls.push({
address: 'https://www.coindega.com/api/0.0.0/#items',
sync: true
});
specter.process(urls, function (err, url, html) {
if (err) {
console.log(err);
} else if (!html) {
console.log('nothing found')
} else {
console.log(html);
}
});
