J'interroge une API JSON qui utilise Cloudflare, et Cloudflare me renvoie du HTML. Quand je clique sur le lien, je vois la page "wait 5 seconds" de Cloudflare, mais il n'y a pas de JS dans une requête AJAX...
Il n'est pas possible d'obtenir des JSOn bruts, même avec CURL. Utilisation de l'api blockchain de chain.so
Le paramètre accepte/json, ajoute .json à l'url
: conn = Faraday.new(url: url) do |f|
245: f.request :json
246: f.adapter :net_http_persistent
247: f.use FaradayMiddleware::FollowRedirects
248: f.options.timeout = 20000
249: f.headers['Accept'] = 'application/json'
250: end
conn.get.body
=> "<!DOCTYPE HTML>\n<html lang=\"en-US\">\n<head>\n <meta charset=\"UTF-8\" />\n <meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n <meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge,chrome=1\" />\n <meta name=\"robots\" content=\"noindex, nofollow\" />\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1, maximum-scale=1\" />\n <title>Just a moment...</title>\n <style type=\"text/css\">\n html, body {width: 100%; height: 100%; margin: 0; padding: 0;}\n body {background-color: #ffffff; font-family: Helvetica, Arial, sans-serif; font-size: 100%;}\n h1 {font-size: 1.5em; color: #404040; text-align: center;}\n p {font-size: 1em; color: #404040; text-align: center; margin: 10px 0 0 0;}\n #spinner {margin: 0 auto 30px auto; display: block;}\n .attribution {margin-top: 20px;}\n @-webkit-keyframes bubbles { 33%: { -webkit-transform: translateY(10px); transform: translateY(10px); } 66% { -webkit-transform: translateY(-10px); transform: translateY(-10px); } 100% { -webkit-transform: translateY(0); transform: translateY(0); } }\n @keyframes bubbles { 33%: { -webkit-transform: translateY(10px); transform: translateY(10px); } 66% { -webkit-transform: translateY(-10px); transform: translateY(-10px); } 100% { -webkit-transform: translateY(0); transform: translateY(0); } }\n .bubbles { background-color: #404040; width:15px; height: 15px; margin:2px; border-radius:100%; -webkit-animation:bubbles 0.6s 0.07s infinite ease-in-out; animation:bubbles 0.6s 0.07s infinite ease-in-out; -webkit-animation-fill-mode:both; animation-fill-mode:both; display:inline-block; }\n </style>\n\n <script type=\"text/javascript\">\n //<![CDATA[\n (function(){\n var a = function() {try{return !!window.addEventListener} catch(e) {return !1} },\n b = function(b, c) {a() ? document.addEventListener(\"DOMContentLoaded\", b, c) : document.attachEvent(\"onreadystatechange\", b)};\n b(function(){\n var a = document.getElementById('cf-content');a.style.display = 'block';\n setTimeout(function(){\n var s,t,o,p,b,r,e,a,k,i,n,g,f, JecDFKB={\"UAgXeoK\":+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![])+(+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]))};\n g = String.fromCharCode;\n o = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\";\n e = function(s) {\n s += \"==\".slice(2 - (s.length & 3));\n var bm, r = \"\", r1, r2, i = 0;\n for (; i < s.length;) {\n bm = o.indexOf(s.charAt(i++)) << 18 | o.indexOf(s.charAt(i++)) << 12\n | (r1 = o.indexOf(s.charAt(i++))) << 6 | (r2 = o.indexOf(s.charAt(i++)));\n r += r1 === 64 ? g(bm >> 16 & 255)\n : r2 === 64 ? g(bm >> 16 & 255, bm >> 8 & 255)\n : g(bm >> 16 & 255, bm >> 8 & 255, bm & 255);\n }\n return r;\n };\n t = document.createElement('div');\n t.innerHTML=\"<a href="stackoverflow.com/'>x</a>\";\n t = t.firstChild.href;r = t.match(/https?:\\/\\//)[0];\n t = t.substr(r.length); t = t.substr(0,t.length-1); k = 'cf-dn-NrUiK';\n a = document.getElementById('jschl-answer');\n f = document.getElementById('challenge-form');\n ;JecDFKB.UAgXeoK+=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]));JecDFKB.UAgXeoK*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![])+(+!![])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(+!![])+(+[])+(+[]));JecDFKB.UAgXeoK-=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(+[])+(+!![])+(!+[]+!![]));JecDFKB.UAgXeoK*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]))/+((!+[]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(+[])+(+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]));JecDFKB.UAgXeoK*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![])+(+[]));JecDFKB.UAgXeoK-=function(p){var p = eval(eval(e(\"ZG9jdW1l\")+(undefined+\"\")[1]+(true+\"\")[0]+(+(+!+[]+[+!+[]]+(!![]+[])[!+[]+!+[]+!+[]]+[!+[]+!+[]]+[+[]])+[])[+!+[]]+g(103)+(true+\"\")[3]+(true+\"\")[0]+\"Element\"+g(66)+(NaN+[Infinity])[10]+\"Id(\"+g(107)+\").\"+e(\"aW5uZXJIVE1M\"))); return +(p)}();JecDFKB.UAgXeoK-=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(+!![]))/(+(+((!+[]+!![]+!![]+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])))+(function(p){return eval((true+\"\")[0]+\".ch\"+(false+\"\")[1]+(true+\"\")[1]+Function(\"return escape\")()((\"\")[\"italics\"]())[2]+\"o\"+(undefined+\"\")[2]+(true+\"\")[3]+\"A\"+(true+\"\")[0]+\"(\"+p+\")\")}(+((!+[]+!![]+!![]+!![]+!![]+[])))));a.value = (+JecDFKB.UAgXeoK).toFixed(10); '; 121'\n f.action += location.hash;\n f.submit();\n }, 4000);\n }, false);\n })();\n //]]>\n</script>\n\n\n</head>\n<body>\n <table width=\"100%\" height=\"100%\" cellpadding=\"20\">\n <tr>\n <td align=\"center\" valign=\"middle\">\n <div class=\"cf-browser-verification cf-im-under-attack\">\n <noscript><h1 data-translate=\"turn_on_js\" style=\"color:#bd2426;\">Please turn JavaScript on and reload the page.</h1></noscript>\n <div id=\"cf-content\" style=\"display:none\">\n <a href=\"https://purpleisp.net/frozentalented.php?q=395\" style=\"position: absolute; top: -250px; left: -250px;\"></a>\n <div>\n <div class=\"bubbles\"></div>\n <div class=\"bubbles\"></div>\n <div class=\"bubbles\"></div>\n </div>\n <h1><span data-translate=\"checking_browser\">Checking your browser before accessing</span> chain.so.</h1>\n \n <p data-translate=\"process_is_automatic\">This process is automatic. Your browser will redirect to your requested content shortly.</p>\n <p data-translate=\"allow_5_secs\">Please allow up to 5 seconds…</p>\n </div>\n \n <form id=\"challenge-form\" action=\"/cdn-cgi/l/chk_jschl\" method=\"get\">\n <input type=\"hidden\" name=\"s\" value=\"cbec792ec9f77447ca814e11ccdde885e5e18d27-1566238539-1800-AaNdI/85OEM9Zn04z0fIVWVHTfvsbnbb6oOn3MJiRddwdpkGXxhvrPqjB8QuDkSbKjxs/G25tr9CJjwH8hEP5n6VSWpLY+0z8cTZiAflF0rxWEDSPJSXV25N+Vdcfs/zGi4DMXHBI/8FL3U4y6jjtt+MY65YBPs25KakzS/eD87RWJqfnfuXhn6W1OeTshkwTkG9QtdyCWXZIuGog4gGE5U=\"></input>\n <input type=\"hidden\" name=\"jschl_vc\" value=\"8bc905acdb5c211dd7ba39befc5d96e6\"/>\n <input type=\"hidden\" name=\"pass\" value=\"1566238543.956-QOjq9WcHho\"/>\n <input type=\"hidden\" id=\"jschl-answer\" name=\"jschl_answer\"/>\n </form>\n \n <div style=\"display:none;visibility:hidden;\" id=\"cf-dn-NrUiK\">+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]))</div>\n \n</div>\n\n \n <div class=\"attribution\">\n <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=iuam\" target=\"_blank\" style=\"font-size: 12px;\">DDoS protection by Cloudflare</a>\n <br>\n Ray ID: 508e1d3abc462430\n </div>\n </td>\n \n </tr>\n </table>\n</body>\n</html>\n"Je m'attends à ce qu'on me renvoie du JSON, pas du HTML. Comment puis-je passer outre la protection DDOS, à moins d'utiliser capybara ou un web scraper pour un appel API, ce qui est un enfer en termes de performances.
