Skip to main content
added 77 characters in body
Source Link
rene
  • 2.8k
  • 2
  • 17
  • 35

StackApi.js

// create only one of these (aka a Singleton)
function StackApi() {

    // How much backoff is needed
    let backoff = 0;

    // awaitable sleep
    function sleep(seconds) {
        return new Promise((resolve)=> {
            setTimeout(resolve, seconds * 1000);
        });
    }

    async function get(url) {

        // if we need to backoff
        while (backoff > 0) {
            // sleep it, async, other backoffs might happen
            const currentBackoff = backoff;
            await sleep(backoff);
            backoff -= currentBackoff;
        } 
        
        // fetch 
        const response = await fetch(url);
        const json = await response.json();
        // check for backoff
        if (json.backoff) {
            // increase backoff to be honered by next caller
            if (backoff < 0) {
                backoff = json.backoff;
            } else {
                backoff += json.backoff;
            }
        }
        // handle errors
        // cache results
        return json;
    }

    return {
        get: get
    }   
}

const instance = new StackApi();

export default instance;

index.js

varimport api2 = new StackApi();from /'./ only ever call this onceStackApi.js'

var baseUrl = 'https://api.stackexchange.com/2.3/posts?order=desc&sort=creation&site=stackoverflow&key=[bring your own key]';

var maxpage = 10;
async function getAnswersPerPage(page) {
    api2.get(baseUrl + "&page=" + page)
        .then( async (data)=> {
            maxpage--;
            console.log('recv: ', (data.items?.length) ?? -1);
            if (data.has_more && maxpage > 0) {
               await getAnswersPerPage(page + 1);
            }
        });
}

getAnswersPerPage(1);
console.log('waiting for results' )
// create only one of these (aka a Singleton)
function StackApi() {

    // How much backoff is needed
    let backoff = 0;

    // awaitable sleep
    function sleep(seconds) {
        return new Promise((resolve)=> {
            setTimeout(resolve, seconds * 1000);
        });
    }

    async function get(url) {

        // if we need to backoff
        while (backoff > 0) {
            // sleep it, async, other backoffs might happen
            const currentBackoff = backoff;
            await sleep(backoff);
            backoff -= currentBackoff;
        } 
        
        // fetch 
        const response = await fetch(url);
        const json = await response.json();
        // check for backoff
        if (json.backoff) {
            // increase backoff to be honered by next caller
            if (backoff < 0) {
                backoff = json.backoff;
            } else {
                backoff += json.backoff;
            }
        }
        // handle errors
        // cache results
        return json;
    }

    return {
        get: get
    }   
}
var api2 = new StackApi(); // only ever call this once

var baseUrl = 'https://api.stackexchange.com/2.3/posts?order=desc&sort=creation&site=stackoverflow&key=[bring your own key]';

var maxpage = 10;
async function getAnswersPerPage(page) {
    api2.get(baseUrl + "&page=" + page)
        .then( async (data)=> {
            maxpage--;
            console.log('recv: ', (data.items?.length) ?? -1);
            if (data.has_more && maxpage > 0) {
               await getAnswersPerPage(page + 1);
            }
        });
}

getAnswersPerPage(1);
console.log('waiting for results' )

StackApi.js

// create only one of these (aka a Singleton)
function StackApi() {

    // How much backoff is needed
    let backoff = 0;

    // awaitable sleep
    function sleep(seconds) {
        return new Promise((resolve)=> {
            setTimeout(resolve, seconds * 1000);
        });
    }

    async function get(url) {

        // if we need to backoff
        while (backoff > 0) {
            // sleep it, async, other backoffs might happen
            const currentBackoff = backoff;
            await sleep(backoff);
            backoff -= currentBackoff;
        } 
        
        // fetch 
        const response = await fetch(url);
        const json = await response.json();
        // check for backoff
        if (json.backoff) {
            // increase backoff to be honered by next caller
            if (backoff < 0) {
                backoff = json.backoff;
            } else {
                backoff += json.backoff;
            }
        }
        // handle errors
        // cache results
        return json;
    }

    return {
        get: get
    }   
}

const instance = new StackApi();

export default instance;

index.js

import api2 from './StackApi.js'

var baseUrl = 'https://api.stackexchange.com/2.3/posts?order=desc&sort=creation&site=stackoverflow&key=[bring your own key]';

var maxpage = 10;
async function getAnswersPerPage(page) {
    api2.get(baseUrl + "&page=" + page)
        .then( async (data)=> {
            maxpage--;
            console.log('recv: ', (data.items?.length) ?? -1);
            if (data.has_more && maxpage > 0) {
               await getAnswersPerPage(page + 1);
            }
        });
}

getAnswersPerPage(1);
console.log('waiting for results' )
Source Link
rene
  • 2.8k
  • 2
  • 17
  • 35

I think a simple singleton that keeps track of the backoff should be enough.

I assume you do use the fetch API or any other async capable module. But the approach is also feasible with just Promises and/or callbacks.

The design of your App will require that you only have one instance that makes the calls to the API.

The general idea is to have one instance that keeps track of the received backoff values (in my implementation I simply add them up). On the next call the get method will first deal with waiting for as many seconds as requested, before making the actual call. Given that JavaScript doesn't like to wait we async/await/Promise our selves with a setTimeout that gets resolved, eventually. Important is to note that we check after waiting again if all backoffs are honered because while we were sleeping other pending network promises might have been resolved, urging for backoffs as well.

In this implementation it is possible to schedule GET calls while a backoff sleep is in progress, leading to these calls to sleep for the same amount and then all call out in quick succession. I didn't bother to cater for that as that is more of a throttling concern. That can be solved similarly in this same class.

Here is the (class like) function that does the work:

// create only one of these (aka a Singleton)
function StackApi() {

    // How much backoff is needed
    let backoff = 0;

    // awaitable sleep
    function sleep(seconds) {
        return new Promise((resolve)=> {
            setTimeout(resolve, seconds * 1000);
        });
    }

    async function get(url) {

        // if we need to backoff
        while (backoff > 0) {
            // sleep it, async, other backoffs might happen
            const currentBackoff = backoff;
            await sleep(backoff);
            backoff -= currentBackoff;
        } 
        
        // fetch 
        const response = await fetch(url);
        const json = await response.json();
        // check for backoff
        if (json.backoff) {
            // increase backoff to be honered by next caller
            if (backoff < 0) {
                backoff = json.backoff;
            } else {
                backoff += json.backoff;
            }
        }
        // handle errors
        // cache results
        return json;
    }

    return {
        get: get
    }   
}

And this is how you would use it:

var api2 = new StackApi(); // only ever call this once

var baseUrl = 'https://api.stackexchange.com/2.3/posts?order=desc&sort=creation&site=stackoverflow&key=[bring your own key]';

var maxpage = 10;
async function getAnswersPerPage(page) {
    api2.get(baseUrl + "&page=" + page)
        .then( async (data)=> {
            maxpage--;
            console.log('recv: ', (data.items?.length) ?? -1);
            if (data.has_more && maxpage > 0) {
               await getAnswersPerPage(page + 1);
            }
        });
}

getAnswersPerPage(1);
console.log('waiting for results' )