Skip to content

Commit

Permalink
support api flags and add page wait flags for puppeteer
Browse files Browse the repository at this point in the history
  • Loading branch information
User Jbelich committed Jan 29, 2019
1 parent 832221b commit 1f2e3e1
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 8 deletions.
4 changes: 1 addition & 3 deletions src/Api/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ import ScrapeTweet from '../TweetScraper';

const router = express.Router();

const timeout = 300;

const twitter = 'https://twitter.com/';

router.get('/hello', (req, res) => {
Expand All @@ -33,7 +31,7 @@ router.get('/tweet/(:user/status/)?:tweet_id([0-9]+)', (req, res) => {
} : {},
pages: 99,
timeline: timelineMode,
replies: timelineMode == 'full' || (doTimeline && isTrue(req.query.replies || false) ? true : false),
replies: timelineMode == 'full' || isTrue(req.query.replies || false),
parents: timelineMode == 'full' || isTrue(req.query.parents || false),
quote: true,
loadWait: 1250,
Expand Down
91 changes: 89 additions & 2 deletions src/App/SearchForm/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ import Paper from '@material-ui/core/Paper';
import Button from '@material-ui/core/Button';
import TextField from '@material-ui/core/TextField';

import Switch from '@material-ui/core/Switch';
import FormGroup from '@material-ui/core/FormGroup';
import FormControlLabel from '@material-ui/core/FormControlLabel';

import CircularProgress from '@material-ui/core/CircularProgress';

const styles = theme => ({
Expand All @@ -23,6 +27,13 @@ const styles = theme => ({
padding: `${theme.spacing.unit * 2}px ${theme.spacing.unit * 3}px ${theme.spacing.unit * 3}px`,
marginTop: theme.spacing.unit * 8,
},
subPaperRoot: {
display: 'flex',
alignItems: 'center',
flexGrow: 1,
padding: `${theme.spacing.unit * 1}px ${theme.spacing.unit * 3}px`,
marginTop: theme.spacing.unit,
},
paperResult: {
display: 'flex',
// alignItems: 'center',
Expand Down Expand Up @@ -60,7 +71,12 @@ class SearchForm extends Component {
state = {
spinner: null,
url: null,
tweetData: null
tweetData: null,
hasTimeline: false,
hasReplies: false,
hasParents: false,
noScreenshot: false,
disabledSwitches: false,
}

doSubmit(event) {
Expand All @@ -86,8 +102,34 @@ class SearchForm extends Component {

// const data = new FormData();
// data.append('file', file, file.name);
let frag = '';
const {hasTimeline, hasReplies, hasParents, noScreenshot} = this.state;
if (hasTimeline == true) {
frag = frag ? (frag + '&') : frag;
// frag = frag + 'timeline=' + (noScreenshot ? 'true' : 'full');
frag = frag + 'timeline=full';

if (noScreenshot == true) {
frag = frag ? (frag + '&') : frag;
frag = frag + 'replies=true&parents=true';
}
} else {
if (hasReplies == true) {
frag = frag ? (frag + '&') : frag;
frag = frag + 'replies=true';
}
if (hasParents == true) {
frag = frag ? (frag + '&') : frag;
frag = frag + 'parents=true';
}
}

if (noScreenshot == true) {
frag = frag ? (frag + '&') : frag;
frag = frag + 'screenshot=false';
}

fetch(fetch_url, {
fetch(fetch_url + (frag ? ('?' + frag) : ''), {

})
.then(res => res.json())
Expand Down Expand Up @@ -125,6 +167,21 @@ class SearchForm extends Component {
});
}

handleSwitch = name => event => {
this.setState({
[name]: event.target.checked
})

if (name == 'hasTimeline') {
this.setState({
disabledSwitches: event.target.checked
});
}
};

// handleSwitch = switch => event => {
// }

render() {
const { classes, theme, children } = this.props;
const { spinner, tweetData } = this.state;
Expand Down Expand Up @@ -153,6 +210,36 @@ class SearchForm extends Component {
</Button>
)}
</Paper>
<Paper className={classes.subPaperRoot} >
<FormGroup row>
<FormControlLabel control={
<Switch value="fullTimeline" color="primary"
checked={this.state.hasTimeline}
onChange={this.handleSwitch('hasTimeline')}
/>
} label="Full Timeline, incl. Reply/Parent Objects" />
<FormControlLabel control={
<Switch value="replies" color="primary"
checked={this.state.hasReplies}
disabled={this.state.disabledSwitches}
onChange={this.handleSwitch('hasReplies')}
/>
} label="Include Reply List" />
<FormControlLabel control={
<Switch value="parents" color="primary"
checked={this.state.hasParents}
disabled={this.state.disabledSwitches}
onChange={this.handleSwitch('hasParents')}
/>
} label="Include Parent List" />
<FormControlLabel control={
<Switch value="no_screenshot" color="secondary"
checked={this.state.noScreenshot}
onChange={this.handleSwitch('noScreenshot')}
/>
} label="No Screenshots" />
</FormGroup>
</Paper>
</form>
{tweetData && (<>
{tweetData.tweetData.screenshot && (<Paper className={classes.paperResult} >
Expand Down
13 changes: 10 additions & 3 deletions src/TweetScraper/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const DEFAULT_OPTIONS = {
parents: true,
quote: true,
loadWait: 1000,
pageWaitMode: 'networkidle0',
doScreenshot: false,
screenshot: {
type: "png",
Expand Down Expand Up @@ -342,7 +343,9 @@ class ScrapeTweet {
const page = await this.getPage();
// console.log(page);

return await page.goto(url)
return await page.goto(url, {
waitUntil: this.options.pageWaitMode || 'load'
})
.then(async response => {
if (!response.ok()) {
throw "Bad Response";
Expand All @@ -366,7 +369,9 @@ class ScrapeTweet {
async getTimeline(url) {
const page = await this.getPage();

return await page.goto(url)
return await page.goto(url, {
waitUntil: this.options.pageWaitMode || 'load'
})
.then(async response => {
if (!response.ok()) {
throw "Bad Response";
Expand Down Expand Up @@ -395,7 +400,9 @@ class ScrapeTweet {
async getSearch(url) {
const page = await this.getPage();

return await page.goto(url)
return await page.goto(url, {
waitUntil: this.options.pageWaitMode || 'load'
})
.then(async response => {
if (!response.ok()) {
throw "Bad Response";
Expand Down

0 comments on commit 1f2e3e1

Please sign in to comment.