Revisions to The /questions endpoint only returns around 30,000 items where 400,000 are expected

added 282 characters in body

Source Link

edited Jan 6, 2023 at 10:42

2.8k
2
17
35

So the work-around should be that the library should not use has_more but instead look at total and the current page times page_size to determine if it needs to fetch another page.:

look at total in the wrapper and the current page times page_size to determine if it needs to fetch another page. This is an expensive operation and should be avoided when possible. The total isn't returned by default so if you want to use it, enable it in your filter first.

look at the items[] array in the wrapper. If that comes back empty (so zero elements or in JavaScript syntax: items.length === 0) you have reached past the last page and can stop fetching more pages.

Unfortunately I can't offer the exact changes that are needed in the library you use but I hope that the code and explanation give enough guidance to implement the workaround.

So the work-around should be that the library should not use has_more but instead look at total and the current page times page_size to determine if it needs to fetch another page. Unfortunately I can't offer the exact changes that are needed in the library you use but I hope that the code and explanation give enough guidance to implement the workaround.

So the work-around should be that the library should not use has_more but instead:

look at total in the wrapper and the current page times page_size to determine if it needs to fetch another page. This is an expensive operation and should be avoided when possible. The total isn't returned by default so if you want to use it, enable it in your filter first.

look at the items[] array in the wrapper. If that comes back empty (so zero elements or in JavaScript syntax: items.length === 0) you have reached past the last page and can stop fetching more pages.

Unfortunately I can't offer the exact changes that are needed in the library you use but I hope that the code and explanation give enough guidance to implement the workaround.

added 105 characters in body

Source Link

edited Jan 2, 2023 at 22:14

rene ♦

2.8k
2
17
35

There is reason to believe this bug is related if not the same as After successfully retrieving 180 pages, the API gracelessly, semi-silently, fails

deleted 193 characters in body

Source Link

edited Jan 2, 2023 at 20:53

rene ♦

2.8k
2
17
35

I have created a stack snippet to demonstrate the problem. (I'm sorry but I'm not an R developer). In the javascript snippet you can choose to useHasMoreOnly (set its value to true) and when you then run the script, it will fetch 313 pages. If you set useHasMoreOnly to false, the code will only look at the total numbercheck of pages (notice that I had to update the filter value to have that field included) to be expected and the current page we're on times page sizeitems were returned. That makes that we happily fetch page 314 and it turns out that works fine.

// set to false to fetch beyond 313 pages
var useHasMoreOnly = true;false;

var key = 'RBeb2Cm7UIYNbN4lwegbaQ((';

var parms = { 
  site: 'stackoverflow',
  tagged:"r", 
  num_pages: 400, //1000000, 
  pagesize:100, 
  filter: "!nOedRLhSVU"  // this filter includes total
  //filter: "!UHY-aKsFJ(KvceZ5uauvQDp9b_ZwAQaEY0KwVy4Czncd97-22tonZWvDXfhmP(X*Baz8J0uC0Q"
};

stack_questions(parms)
  .then( (df_questions_r) => {
    document.getElementsByTagName('body')[0].textContent = `fetched ${df_questions_r.length} posts`;  // JSON.stringify(df_questions_r);
  })
  .catch(console.log);

// library magic
function stack_questions(opt) {
   var url = 'https://api.stackexchange.com/2.3/questions';
   var items = [];
   return new Promise( (resolve, reject) => {
   
    function getNext(next, url,key,opt, page) {
    
    }
    
    function get(url, key, opt, page) { 
        var num_pages = opt.num_pages || 1;
        var localOpt = Object.assign({ key:key, page:page}, opt);
        var optList = Object
          .keys(localOpt)
          .filter( k=> k !== 'num_pages')
          .reduce( function(p,c) { return `${p}&${c}=${localOpt[c]}` }, "");
        var nexturl = url + '?' + optList.substring(1);
        fetch(nexturl)
          .then((resp) => resp.json())
          .then((json) => {
             for(const item of json.items || []) {
               items.push(item)
             }
             return json
           })
         .then((json) => {
             if (json.backoff) {
                console.log('backoff for ', json.backoff , page);
             }
             var waitMs = (json.backoff || 1 ) * 1000;
             
             var local_count = json.items.length;
             var has_more = json.has_more;
             var has_pages_left = page < num_pages;
             var has_records_left = local_countjson.items <&& json.total;items.length > 0; 
             // do we stop or do we get the next page? 
             if (useHasMoreOnly) {
                // if the the api says it has more
                // that
                if (has_more) {
                  setTimeout(get(url,key,opt, page + 1), waitMs); 
                } else {
                    resolve(items);
                }
             } else {
               // we do not trust has_more so we
               // use total and our own counts to decide
               // to get another page
               if (has_pages_left && has_records_left) {
                 setTimeout(get(url,key,opt, page + 1), waitMs); 
                } else {
                    resolve(items);
                }
             }
          })
          .catch(reject);
     }
     
     get(url, key, opt, 1);
   });
}

I have created a stack snippet to demonstrate the problem. (I'm sorry but I'm not an R developer). In the javascript snippet you can choose to useHasMoreOnly (set its value to true) and when you then run the script, it will fetch 313 pages. If you set useHasMoreOnly to false, the code will only look at the total number of pages (notice that I had to update the filter value to have that field included) to be expected and the current page we're on times page size. That makes that we happily fetch page 314 and it turns out that works fine.

// set to false to fetch beyond 313 pages
var useHasMoreOnly = true;

var key = 'RBeb2Cm7UIYNbN4lwegbaQ((';

var parms = { 
  site: 'stackoverflow',
  tagged:"r", 
  num_pages: 400, //1000000, 
  pagesize:100, 
  filter: "!nOedRLhSVU"  // this filter includes total
  //filter: "!UHY-aKsFJ(KvceZ5uauvQDp9b_ZwAQaEY0KwVy4Czncd97-22tonZWvDXfhmP(X*Baz8J0uC0Q"
};

stack_questions(parms)
  .then( (df_questions_r) => {
    document.getElementsByTagName('body')[0].textContent = `fetched ${df_questions_r.length} posts`;  // JSON.stringify(df_questions_r);
  })
  .catch(console.log);

// library magic
function stack_questions(opt) {
   var url = 'https://api.stackexchange.com/2.3/questions';
   var items = [];
   return new Promise( (resolve, reject) => {
   
    function getNext(next, url,key,opt, page) {
    
    }
    
    function get(url, key, opt, page) { 
        var num_pages = opt.num_pages || 1;
        var localOpt = Object.assign({ key:key, page:page}, opt);
        var optList = Object
          .keys(localOpt)
          .filter( k=> k !== 'num_pages')
          .reduce( function(p,c) { return `${p}&${c}=${localOpt[c]}` }, "");
        var nexturl = url + '?' + optList.substring(1);
        fetch(nexturl)
          .then((resp) => resp.json())
          .then((json) => {
             for(const item of json.items || []) {
               items.push(item)
             }
             return json
           })
         .then((json) => {
             if (json.backoff) {
                console.log('backoff for ', json.backoff , page);
             }
             var waitMs = (json.backoff || 1 ) * 1000;
             
             var local_count = json.items.length;
             var has_more = json.has_more;
             var has_pages_left = page < num_pages;
             var has_records_left = local_count < json.total; 
             // do we stop or do we get the next page? 
             if (useHasMoreOnly) {
                // if the the api says it has more
                // that
                if (has_more) {
                  setTimeout(get(url,key,opt, page + 1), waitMs); 
                } else {
                    resolve(items);
                }
             } else {
               // we do not trust has_more so we
               // use total and our own counts to decide
               // to get another page
               if (has_pages_left && has_records_left) {
                 setTimeout(get(url,key,opt, page + 1), waitMs); 
                } else {
                    resolve(items);
                }
             }
          })
          .catch(reject);
     }
     
     get(url, key, opt, 1);
   });
}

I have created a stack snippet to demonstrate the problem. (I'm sorry but I'm not an R developer). In the javascript snippet you can choose to useHasMoreOnly (set its value to true) and when you then run the script, it will fetch 313 pages. If you set useHasMoreOnly to false, the code will only check of items were returned. That makes that we happily fetch page 314 and it turns out that works fine.

// set to false to fetch beyond 313 pages
var useHasMoreOnly = false;

var key = 'RBeb2Cm7UIYNbN4lwegbaQ((';

var parms = { 
  site: 'stackoverflow',
  tagged:"r", 
  num_pages: 400, //1000000, 
  pagesize:100, 
  filter: "!UHY-aKsFJ(KvceZ5uauvQDp9b_ZwAQaEY0KwVy4Czncd97-22tonZWvDXfhmP(X*Baz8J0uC0Q"
};

stack_questions(parms)
  .then( (df_questions_r) => {
    document.getElementsByTagName('body')[0].textContent = `fetched ${df_questions_r.length} posts`;  // JSON.stringify(df_questions_r);
  })
  .catch(console.log);

// library magic
function stack_questions(opt) {
   var url = 'https://api.stackexchange.com/2.3/questions';
   var items = [];
   return new Promise( (resolve, reject) => {
   
    function getNext(next, url,key,opt, page) {
    
    }
    
    function get(url, key, opt, page) { 
        var num_pages = opt.num_pages || 1;
        var localOpt = Object.assign({ key:key, page:page}, opt);
        var optList = Object
          .keys(localOpt)
          .filter( k=> k !== 'num_pages')
          .reduce( function(p,c) { return `${p}&${c}=${localOpt[c]}` }, "");
        var nexturl = url + '?' + optList.substring(1);
        fetch(nexturl)
          .then((resp) => resp.json())
          .then((json) => {
             for(const item of json.items || []) {
               items.push(item)
             }
             return json
           })
         .then((json) => {
             if (json.backoff) {
                console.log('backoff for ', json.backoff , page);
             }
             var waitMs = (json.backoff || 1 ) * 1000;
             
             var local_count = json.items.length;
             var has_more = json.has_more;
             var has_pages_left = page < num_pages;
             var has_records_left = json.items && json.items.length > 0; 
             // do we stop or do we get the next page? 
             if (useHasMoreOnly) {
                // if the the api says it has more
                // that
                if (has_more) {
                  setTimeout(get(url,key,opt, page + 1), waitMs); 
                } else {
                    resolve(items);
                }
             } else {
               // we do not trust has_more so we
               // use total and our own counts to decide
               // to get another page
               if (has_pages_left && has_records_left) {
                 setTimeout(get(url,key,opt, page + 1), waitMs); 
                } else {
                    resolve(items);
                }
             }
          })
          .catch(reject);
     }
     
     get(url, key, opt, 1);
   });
}

Source Link

created Jan 2, 2023 at 20:35

rene ♦

2.8k
2
17
35

Loading

Stack Exchange Network

Return to Answer