Jump to content

Curl and Godaddy


Rusty3

Recommended Posts

For the first time in my life, I can't open a webpage with curl:

 

   $ch = curl_init(); 
    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
    //curl_setopt($ch, CURLOPT_SSLVERSION, 3);
    curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.02; Windows NT 5.0)");
    curl_setopt($ch, CURLOPT_COOKIEJAR, dirname(__FILE__).'/godaddycokie.txt');
    curl_setopt($ch, CURLOPT_COOKIEFILE, dirname(__FILE__).'/godaddycokie.txt');
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); 
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); 
    $url = 'https://auctions.godaddy.com';
    curl_setopt($ch, CURLOPT_REFERER, $url);
    echo curl_exec($ch); 
    

 

Any clues? Some Javascript trick?!

Link to comment
Share on other sites

I did a http://google.com search, and found this page.

I use GoDaddy Hosting  for this site. A few days ago I tried to use the curl library. Curl was there - but all my calls timed out. I tried different approaches - but all failed. After some google searches I found out why - GoDaddy is using a proxy server.

 

So if you are using GoDaddy hosting and what to use curl, you should include these lines in the script...

 

<?php
curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
curl_setopt($ch, CURLOPT_PROXY,"http://64.202.165.130:3128");
?>

 

 

Link to comment
Share on other sites

Always build in error checking . .

 

<?php

$ch = curl_init(); 
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
//curl_setopt($ch, CURLOPT_SSLVERSION, 3);
curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.02; Windows NT 5.0)");
curl_setopt($ch, CURLOPT_COOKIEJAR, dirname(__FILE__).'/godaddycokie.txt');
curl_setopt($ch, CURLOPT_COOKIEFILE, dirname(__FILE__).'/godaddycokie.txt');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); 
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); 
$url = 'https://auctions.godaddy.com';
curl_setopt($ch, CURLOPT_REFERER, $url);
$response = curl_exec($ch); 

if ($response === false)
{
echo curl_error($ch);
}
else
{
echo "Response OK";
}

?>

Link to comment
Share on other sites

Thanks spfoonnewb, the snippet I posted was wrong, but that was not the problem.

The problem is that now Godaddy inserts a random string and a random parameter ("rnd" and "nblPLWxHGVnEAfbPcBZp") when searching their auctions page. I hit godaddy.com first (to create the session and store it on the same cookie) but it doesn't work.

 

Any ideas on how to simulate the behavior they do on the complex Javascripts they run?

 

 

 

    $ch = curl_init(); 
    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
    curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.02; Windows NT 5.0)");
    curl_setopt($ch, CURLOPT_COOKIEJAR, dirname(__FILE__).'/godaddycokie.txt');
    curl_setopt($ch, CURLOPT_COOKIEFILE, dirname(__FILE__).'/godaddycokie.txt');
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); 
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
    curl_setopt($ch, CURLOPT_URL, 'https://auctions.godaddy.com');
    curl_exec($ch);
    curl_setopt($ch, CURLOPT_POST, 1);          
    curl_setopt($ch, CURLOPT_POSTFIELDS, 't=16&action=search&hidAdvSearch=ddlAdvKeyword:3|txtKeyword:|ddlCharacters:0|txtCharacters:|txtMinTraffic:|txtMaxTraffic:|txtMinPrice:|txtMaxPrice:|ddlCategories:0|chkAddBuyNow:false|chkAddFeatured:false|chkAddDash:true|chkAddDigit:true|chkAddWeb:false|chkAddAppr:false|chkAddInv:false|ddlPattern1:-|ddlPattern2:-|ddlPattern3:-|ddlPattern4:-|chkSaleOffer:false|chkSalePublic:false|chkSaleExpired:true|chkSaleCloseouts:false|chkSaleUsed:false|chkAddOnSale:false|ddlAdvBids:0|txtBids:|txtAuctionID:|chkTLD1:false|chkTLD2:false|chkTLD3:false|chkTLD4:false|chkTLD5:false|chkTLD6:false|chkTLD7:false|chkTLD233:false|chkTLD240:false|chkTLD251:false|chkTLD292:false|chkTLD69:false|chkTLD22:false|chkTLD85:false|chkTLD396:false|chkTLD51:false|chkTLD50:false|chkTLD465:false|chkTLD466:false|chkTLD80:false|chkTLD366:false|chkTLD62:false|chkTLD205:false|ddlDateOffset:&rpp=15&page=1&pageMode=2&rnd=0.4006484983301911&nblPLWxHGVnEAfbPcBZp=74091bb5e79d499aa8e591b57ed932bb');
    curl_setopt($ch, CURLOPT_URL, 'https://auctions.godaddy.com/trpSearchResults.aspx');    
    $response = curl_exec($ch); 
    if ($response === false)
{   echo curl_error($ch);}
else
{ echo "Response OK:$response";}

Link to comment
Share on other sites

  • 2 weeks later...
  • 1 month later...

What's actually happening that you can't get in?

 

This basically works for me...

<?php
echo get_final_url("http://auctions.godaddy.com");
function get_final_url( $url, $timeout = 5 ) {
    $url = str_replace( "&", "&", urldecode(trim($url)) );
    $cookie = tempnam ("/tmp", "CURLCOOKIE");
    $ch = curl_init();
    curl_setopt( $ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1" );
    curl_setopt( $ch, CURLOPT_URL, $url );
    curl_setopt( $ch, CURLOPT_COOKIEJAR, $cookie );
    curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, true );
    curl_setopt( $ch, CURLOPT_ENCODING, "" );
    curl_setopt( $ch, CURLOPT_RETURNTRANSFER, true );
    curl_setopt( $ch, CURLOPT_SSL_VERIFYPEER, false);
    curl_setopt( $ch, CURLOPT_AUTOREFERER, true );
    curl_setopt( $ch, CURLOPT_CONNECTTIMEOUT, $timeout );
    curl_setopt( $ch, CURLOPT_TIMEOUT, $timeout );
    curl_setopt( $ch, CURLOPT_MAXREDIRS, 10 );
    $content = curl_exec( $ch );
    $response = curl_getinfo( $ch );
    if ($response === false) {
        echo curl_error($ch);
    } else { 
        return $content;
    }
    curl_close ( $ch );
}
?>

Link to comment
Share on other sites

Thanks quasiman. The problem is that I can't do a successful CURL POST request.

 

What's actually happening that you can't get in?

 

This basically works for me...

<?php
echo get_final_url("http://auctions.godaddy.com");
function get_final_url( $url, $timeout = 5 ) {
    $url = str_replace( "&", "&", urldecode(trim($url)) );
    $cookie = tempnam ("/tmp", "CURLCOOKIE");
    $ch = curl_init();
    curl_setopt( $ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1" );
    curl_setopt( $ch, CURLOPT_URL, $url );
    curl_setopt( $ch, CURLOPT_COOKIEJAR, $cookie );
    curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, true );
    curl_setopt( $ch, CURLOPT_ENCODING, "" );
    curl_setopt( $ch, CURLOPT_RETURNTRANSFER, true );
    curl_setopt( $ch, CURLOPT_SSL_VERIFYPEER, false);
    curl_setopt( $ch, CURLOPT_AUTOREFERER, true );
    curl_setopt( $ch, CURLOPT_CONNECTTIMEOUT, $timeout );
    curl_setopt( $ch, CURLOPT_TIMEOUT, $timeout );
    curl_setopt( $ch, CURLOPT_MAXREDIRS, 10 );
    $content = curl_exec( $ch );
    $response = curl_getinfo( $ch );
    if ($response === false) {
        echo curl_error($ch);
    } else { 
        return $content;
    }
    curl_close ( $ch );
}
?>

Link to comment
Share on other sites

javascript and/or cookie/referer/other tricks. i have only recently been stumped by a website i was trying to scrape. i could not get the data via curl, no matter what i tried. if you find a way to use PHP to parse and execute Javascript, i would like your autograph.

Link to comment
Share on other sites

  • 5 months later...

I noticed I could curl it and redirects, I use a code to handle the javascript redirects.

 

Inserted url:

https://auctions.godaddy.com/

Redirected url:

http://auctions.godaddy.com/trpHome.aspx

Date: 2011-04-12 20:35:01

Title: GoDaddy Domain Auctions, Buy and Sell Domain Names

Description: GoDaddy Auctions is the place to go for great domain names that are expiring or have been put up for auction. GoDaddy Auctions makes it easy to get the domain name you have been looking for.

Keywords: GoDaddy, GoDaddy Auctions, Domain Auctions, Domain Auction

 

I'll be a nice sport and relinquish my code.

Here's my complete curl code for connecting and displays the html as an example.

I disabled cookies because I was collecting too many of them.

<?php
$url = "https://auctions.godaddy.com/";
/*connect to the url using curl to see if exists and get the information*/
        //$cookie = tempnam('tmp','cookie');
        //$cookie_file_path = "tmp/";
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        //curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie);
        //curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_file_path);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3');
        curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
        curl_setopt($ch, CURLOPT_TIMEOUT, 15);
        curl_setopt($ch, CURLOPT_MAXREDIRS, 15);
        curl_setopt($ch, CURLOPT_HEADER, 1);
        curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1);
        curl_setopt($ch, CURLOPT_AUTOREFERER, true);
        curl_setopt ($ch, CURLOPT_FILETIME, 1);
        curl_setopt ($ch, CURLOPT_FOLLOWLOCATION, 1);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
        curl_setopt($ch, CURLOPT_ENCODING , "");
        $curl_session = curl_init();
        //curl_setopt($curl_session, CURLOPT_COOKIEJAR, $cookie);
        //curl_setopt($curl_session, CURLOPT_COOKIEFILE, $cookie_file_path);
        curl_setopt($curl_session, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3');
        curl_setopt($curl_session, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
        curl_setopt($curl_session, CURLOPT_ENCODING , "");
        curl_setopt($curl_session, CURLOPT_TIMEOUT, 15);
        curl_setopt($curl_session, CURLOPT_HEADER, 1);
        curl_setopt($curl_session, CURLOPT_SSL_VERIFYPEER, FALSE);
        curl_setopt($curl_session, CURLOPT_HEADER, true);
        curl_setopt($curl_session, CURLOPT_MAXREDIRS, 15);
        curl_setopt($curl_session, CURLOPT_RETURNTRANSFER, true);
        curl_setopt( $curl_session, CURLOPT_AUTOREFERER, true );
        curl_setopt ($curl_session, CURLOPT_HTTPGET, true);
        curl_setopt($curl_session, CURLOPT_URL, $url);
        $string = mysql_real_escape_string(curl_exec($curl_session));
        $html = mysql_real_escape_string(curl_exec ($ch));
        $info = curl_getinfo($ch);
        /*curl response check and to resolve url to the actual location*/
        $response = curl_getinfo( $ch );
        if ($response['http_code'] == 301 || $response['http_code'] == 302) {
            ini_set("user_agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3");
            $headers = get_headers($response['url']);

            $location = "";
            foreach( $headers as $value ) {
                if ( substr( strtolower($value), 0, 9 ) == "location:" )
                    return get_final_url( trim( substr( $value, 9, strlen($value) ) ) );
            }
        }

        if (    preg_match("/window\.location\.replace\('(.*)'\)/i", $con_addtent, $value) ||
        preg_match("/window\.location\=[\"'](.*)[\"']/i", $con_addtent, $value) ||
        preg_match("/location\.href\=[\"'](.*)[\"']/i", $con_addtent, $value)
)
       {
            $finalurl = get_final_url($value[1]);
        }
        else {

            $finalurl = $response['url'];
        }
        
        $html = curl_exec($ch);
        $header = "Location: ";
        
echo "$finalurl<br />";

echo $html;

?>

Link to comment
Share on other sites

Archived

This topic is now archived and is closed to further replies.

×
×
  • Create New...

Important Information

We have placed cookies on your device to help make this website better. You can adjust your cookie settings, otherwise we'll assume you're okay to continue.