#!/usr/bin/env python
"""
Python wrapper for Google web APIs
This module allows you to access Google's web APIs through SOAP,
to do things like search Google and get the results programmatically.
Described U{here }
You need a Google-provided license key to use these services.
Follow the link above to get one. These functions will look in
several places (in this order) for the license key:
- the "license_key" argument of each function
- the module-level LICENSE_KEY variable (call setLicense once to set it)
- an environment variable called GOOGLE_LICENSE_KEY
- a file called ".googlekey" in the current directory
- a file called "googlekey.txt" in the current directory
- a file called ".googlekey" in your home directory
- a file called "googlekey.txt" in your home directory
- a file called ".googlekey" in the same directory as google.py
- a file called "googlekey.txt" in the same directory as google.py
Sample usage::
>>> import google
>>> google.setLicense('...') # must get your own key!
>>> data = google.doGoogleSearch('python')
>>> data.meta.searchTime
0.043221000000000002
>>> data.results[0].URL
'http://www.python.org/'
>>> data.results[0].title
'Python Language Website'
@newfield contrib: Contributors
@author: Mark Pilgrim
@author: Brian Landers
@license: Python
@version: 0.6
@contrib: David Ascher, for the install script
@contrib: Erik Max Francis, for the command line interface
@contrib: Michael Twomey, for HTTP proxy support
@contrib: Mark Recht, for patches to support SOAPpy
"""
__author__ = "Mark Pilgrim (f8dy@diveintomark.org)"
__version__ = "0.6"
__cvsversion__ = "$Revision: 1.5 $"[11:-2]
__date__ = "$Date: 2004/02/25 23:46:07 $"[7:-2]
__copyright__ = "Copyright (c) 2002 Mark Pilgrim"
__license__ = "Python"
__credits__ = """David Ascher, for the install script
Erik Max Francis, for the command line interface
Michael Twomey, for HTTP proxy support"""
import os, sys, getopt
import GoogleSOAPFacade
LICENSE_KEY = None
HTTP_PROXY = None
#
# Constants
#
_url = 'http://api.google.com/search/beta2'
_namespace = 'urn:GoogleSearch'
_googlefile1 = ".googlekey"
_googlefile2 = "googlekey.txt"
_false = GoogleSOAPFacade.false
_true = GoogleSOAPFacade.true
_licenseLocations = (
( lambda key: key,
'passed to the function in license_key variable' ),
( lambda key: LICENSE_KEY,
'module-level LICENSE_KEY variable (call setLicense to set it)' ),
( lambda key: os.environ.get( 'GOOGLE_LICENSE_KEY', None ),
'an environment variable called GOOGLE_LICENSE_KEY' ),
( lambda key: _contentsOf( os.getcwd(), _googlefile1 ),
'%s in the current directory' % _googlefile1),
( lambda key: _contentsOf( os.getcwd(), _googlefile2 ),
'%s in the current directory' % _googlefile2),
( lambda key: _contentsOf( os.environ.get( 'HOME', '' ), _googlefile1 ),
'%s in your home directory' % _googlefile1),
( lambda key: _contentsOf( os.environ.get( 'HOME', '' ), _googlefile2 ),
'%s in your home directory' % _googlefile2 ),
( lambda key: _contentsOf( _getScriptDir(), _googlefile1 ),
'%s in the google.py directory' % _googlefile1 ),
( lambda key: _contentsOf( _getScriptDir(), _googlefile2 ),
'%s in the google.py directory' % _googlefile2 )
)
## ----------------------------------------------------------------------
## Exceptions
## ----------------------------------------------------------------------
class NoLicenseKey(Exception):
"""
Thrown when the API is unable to find a valid license key.
"""
pass
## ----------------------------------------------------------------------
## administrative functions (non-API)
## ----------------------------------------------------------------------
def _version():
"""
Display a formatted version string for the module
"""
print """PyGoogle %(__version__)s
%(__copyright__)s
released %(__date__)s
Thanks to:
%(__credits__)s""" % globals()
def _usage():
"""
Display usage information for the command-line interface
"""
program = os.path.basename(sys.argv[0])
print """Usage: %(program)s [options] [querytype] query
options:
-k, --key= Google license key (see important note below)
-1, -l, --lucky show only first hit
-m, --meta show meta information
-r, --reverse show results in reverse order
-x, --proxy= use HTTP proxy
-h, --help print this help
-v, --version print version and copyright information
-t, --test run test queries
querytype:
-s, --search= search (default)
-c, --cache= retrieve cached page
-p, --spelling= check spelling
IMPORTANT NOTE: all Google functions require a valid license key;
visit http://www.google.com/apis/ to get one. %(program)s will look in
these places (in order) and use the first license key it finds:
* the key specified on the command line""" % vars()
for get, location in _licenseLocations[2:]:
print " *", location
## ----------------------------------------------------------------------
## utility functions (API)
## ----------------------------------------------------------------------
def setLicense(license_key):
"""
Set the U{Google APIs } license key
@param license_key: The new key to use
@type license_key: String
@todo: validate the key?
"""
global LICENSE_KEY
LICENSE_KEY = license_key
def getLicense(license_key = None):
"""
Get the U{Google APIs } license key
The key can be read from any number of locations. See the module-leve
documentation for the search order.
@return: the license key
@rtype: String
@raise NoLicenseKey: if no valid key could be found
"""
for get, location in _licenseLocations:
rc = get(license_key)
if rc: return rc
_usage()
raise NoLicenseKey, 'get a license key at http://www.google.com/apis/'
def setProxy(http_proxy):
"""
Set the HTTP proxy to be used when accessing Google
@param http_proxy: the proxy to use
@type http_proxy: String
@todo: validiate the input?
"""
global HTTP_PROXY
HTTP_PROXY = http_proxy
def getProxy(http_proxy = None):
"""
Get the HTTP proxy we use for accessing Google
@return: the proxy
@rtype: String
"""
return http_proxy or HTTP_PROXY
def _contentsOf(dirname, filename):
filename = os.path.join(dirname, filename)
if not os.path.exists(filename): return None
fsock = open(filename)
contents = fsock.read()
fsock.close()
return contents
def _getScriptDir():
if __name__ == '__main__':
return os.path.abspath(os.path.dirname(sys.argv[0]))
else:
return os.path.abspath(os.path.dirname(sys.modules[__name__].__file__))
def _marshalBoolean(value):
if value:
return _true
else:
return _false
def _getRemoteServer( http_proxy ):
return GoogleSOAPFacade.getProxy( _url, _namespace, http_proxy )
## ----------------------------------------------------------------------
## search results classes
## ----------------------------------------------------------------------
class _SearchBase:
def __init__(self, params):
for k, v in params.items():
if isinstance(v, GoogleSOAPFacade.structType):
v = GoogleSOAPFacade.toDict( v )
try:
if isinstance(v[0], GoogleSOAPFacade.structType):
v = [ SOAPProxy.toDict( node ) for node in v ]
except:
pass
self.__dict__[str(k)] = v
## ----------------------------------------------------------------------
class SearchResultsMetaData(_SearchBase):
"""
Container class for metadata about a given search query's results.
@ivar documentFiltering: is duplicate page filtering active?
@ivar searchComments: human-readable informational message
example::
"'the' is a very common word and was not included in your search"
@ivar estimatedTotalResultsCount: estimated total number of results
for this query.
@ivar estimateIsExact: is estimatedTotalResultsCount an exact value?
@ivar searchQuery: search string that initiated this search
@ivar startIndex: index of the first result returned (zero-based)
@ivar endIndex: index of the last result returned (zero-based)
@ivar searchTips: human-readable informational message on how to better
use Google.
@ivar directoryCategories: list of categories for the search results
This field is a list of dictionaries, like so::
{ 'fullViewableName': 'the Open Directory category',
'specialEncoding': 'encoding scheme of this directory category'
}
@ivar searchTime: total search time, in seconds
"""
pass
## ----------------------------------------------------------------------
class SearchResult(_SearchBase):
"""
Encapsulates the results from a search.
@ivar URL: URL
@ivar title: title (HTML)
@ivar snippet: snippet showing query context (HTML
@ivar cachedSize: size of cached version of this result, (KB)
@ivar relatedInformationPresent: is the "related:" keyword supported?
Flag indicates that the "related:" keyword is supported for this URL
@ivar hostName: used when filtering occurs
When filtering occurs, a maximum of two results from any given
host is returned. When this occurs, the second resultElement
that comes from that host contains the host name in this parameter.
@ivar directoryCategory: Open Directory category information
This field is a dictionary with the following values::
{ 'fullViewableName': 'the Open Directory category',
'specialEncoding' : 'encoding scheme of this directory category'
}
@ivar directoryTitle: Open Directory title of this result (or blank)
@ivar summary: Open Directory summary for this result (or blank)
"""
pass
## ----------------------------------------------------------------------
class SearchReturnValue:
"""
complete search results for a single query
@ivar meta: L{SearchResultsMetaData} instance for this query
@ivar results: list of L{SearchResult} objects for this query
"""
def __init__( self, metadata, results ):
self.meta = metadata
self.results = results
## ----------------------------------------------------------------------
## main functions
## ----------------------------------------------------------------------
def doGoogleSearch( q, start = 0, maxResults = 10, filter = 1,
restrict='', safeSearch = 0, language = '',
inputencoding = '', outputencoding = '',\
license_key = None, http_proxy = None ):
"""
Search Google using the SOAP API and return the results.
You need a license key to call this function; see the
U{Google APIs } site to get one.
Then you can either pass it to this function every time, or
set it globally; see the L{google} module-level docs for details.
See U{http://www.google.com/help/features.html}
for examples of advanced features. Anything that works at the
Google web site will work as a query string in this method.
You can use the C{start} and C{maxResults} parameters to page
through multiple pages of results. Note that 'maxResults' is
currently limited by Google to 10.
See the API reference for more advanced examples and a full list of
country codes and topics for use in the C{restrict} parameter, along
with legal values for the C{language}, C{inputencoding}, and
C{outputencoding} parameters.
You can download the API documentation
U{http://www.google.com/apis/download.html }.
@param q: search string.
@type q: String
@param start: (optional) zero-based index of first desired result.
@type start: int
@param maxResults: (optional) maximum number of results to return.
@type maxResults: int
@param filter: (optional) flag to request filtering of similar results
@type filter: int
@param restrict: (optional) restrict results by country or topic.
@type restrict: String
@param safeSearch: (optional)
@type safeSearch: int
@param language: (optional)
@type language: String
@param inputencoding: (optional)
@type inputencoding: String
@param outputencoding: (optional)
@type outputencoding: String
@param license_key: (optional) the Google API license key to use
@type license_key: String
@param http_proxy: (optional) the HTTP proxy to use for talking to Google
@type http_proxy: String
@return: the search results encapsulated in an object
@rtype: L{SearchReturnValue}
"""
license_key = getLicense( license_key )
http_proxy = getProxy( http_proxy )
remoteserver = _getRemoteServer( http_proxy )
filter = _marshalBoolean( filter )
safeSearch = _marshalBoolean( safeSearch )
data = remoteserver.doGoogleSearch( license_key, q, start, maxResults,
filter, restrict, safeSearch,
language, inputencoding,
outputencoding )
metadata = GoogleSOAPFacade.toDict( data )
del metadata["resultElements"]
metadata = SearchResultsMetaData( metadata )
results = [ SearchResult( GoogleSOAPFacade.toDict( node ) ) \
for node in data.resultElements ]
return SearchReturnValue( metadata, results )
## ----------------------------------------------------------------------
def doGetCachedPage( url, license_key = None, http_proxy = None ):
"""
Retrieve a page from the Google cache.
You need a license key to call this function; see the
U{Google APIs } site to get one.
Then you can either pass it to this function every time, or
set it globally; see the L{google} module-level docs for details.
@param url: full URL to the page to retrieve
@type url: String
@param license_key: (optional) the Google API key to use
@type license_key: String
@param http_proxy: (optional) the HTTP proxy server to use
@type http_proxy: String
@return: full text of the cached page
@rtype: String
"""
license_key = getLicense( license_key )
http_proxy = getProxy( http_proxy )
remoteserver = _getRemoteServer( http_proxy )
return remoteserver.doGetCachedPage( license_key, url )
## ----------------------------------------------------------------------
def doSpellingSuggestion( phrase, license_key = None, http_proxy = None ):
"""
Get spelling suggestions from Google
You need a license key to call this function; see the
U{Google APIs } site to get one.
Then you can either pass it to this function every time, or
set it globally; see the L{google} module-level docs for details.
@param phrase: word or phrase to spell-check
@type phrase: String
@param license_key: (optional) the Google API key to use
@type license_key: String
@param http_proxy: (optional) the HTTP proxy to use
@type http_proxy: String
@return: text of any suggested replacement, or None
"""
license_key = getLicense( license_key )
http_proxy = getProxy( http_proxy)
remoteserver = _getRemoteServer( http_proxy )
return remoteserver.doSpellingSuggestion( license_key, phrase )
## ----------------------------------------------------------------------
## functional test suite (see googletest.py for unit test suite)
## ----------------------------------------------------------------------
def _test():
"""
Run functional test suite.
"""
try:
getLicense(None)
except NoLicenseKey:
return
print "Searching for Python at google.com..."
data = doGoogleSearch( "Python" )
_output( data, { "func": "doGoogleSearch"} )
print "\nSearching for 5 _French_ pages about Python, "
print "encoded in ISO-8859-1..."
data = doGoogleSearch( "Python", language = 'lang_fr',
outputencoding = 'ISO-8859-1',
maxResults = 5 )
_output( data, { "func": "doGoogleSearch" } )
phrase = "Pyhton programming languager"
print "\nTesting spelling suggestions for '%s'..." % phrase
data = doSpellingSuggestion( phrase )
_output( data, { "func": "doSpellingSuggestion" } )
## ----------------------------------------------------------------------
## Command-line interface
## ----------------------------------------------------------------------
class _OutputFormatter:
def boil(self, data):
if type(data) == type(u""):
return data.encode("ISO-8859-1", "replace")
else:
return data
class _TextOutputFormatter(_OutputFormatter):
def common(self, data, params):
if params.get("showMeta", 0):
meta = data.meta
for category in meta.directoryCategories:
print "directoryCategory: %s" % \
self.boil(category["fullViewableName"])
for attr in [node for node in dir(meta) if \
node <> "directoryCategories" and node[:2] <> '__']:
print "%s:" % attr, self.boil(getattr(meta, attr))
def doGoogleSearch(self, data, params):
results = data.results
if params.get("feelingLucky", 0):
results = results[:1]
if params.get("reverseOrder", 0):
results.reverse()
for result in results:
for attr in dir(result):
if attr == "directoryCategory":
print "directoryCategory:", \
self.boil(result.directoryCategory["fullViewableName"])
elif attr[:2] <> '__':
print "%s:" % attr, self.boil(getattr(result, attr))
print
self.common(data, params)
def doGetCachedPage(self, data, params):
print data
self.common(data, params)
doSpellingSuggestion = doGetCachedPage
def _makeFormatter(outputFormat):
classname = "_%sOutputFormatter" % outputFormat.capitalize()
return globals()[classname]()
def _output(results, params):
formatter = _makeFormatter(params.get("outputFormat", "text"))
outputmethod = getattr(formatter, params["func"])
outputmethod(results, params)
def main(argv):
"""
Command-line interface.
"""
if not argv:
_usage()
return
q = None
func = None
http_proxy = None
license_key = None
feelingLucky = 0
showMeta = 0
reverseOrder = 0
runTest = 0
outputFormat = "text"
try:
opts, args = getopt.getopt(argv, "s:c:p:k:lmrx:hvt1",
["search=", "cache=", "spelling=", "key=", "lucky", "meta",
"reverse", "proxy=", "help", "version", "test"])
except getopt.GetoptError:
_usage()
sys.exit(2)
for opt, arg in opts:
if opt in ("-s", "--search"):
q = arg
func = "doGoogleSearch"
elif opt in ("-c", "--cache"):
q = arg
func = "doGetCachedPage"
elif opt in ("-p", "--spelling"):
q = arg
func = "doSpellingSuggestion"
elif opt in ("-k", "--key"):
license_key = arg
elif opt in ("-l", "-1", "--lucky"):
feelingLucky = 1
elif opt in ("-m", "--meta"):
showMeta = 1
elif opt in ("-r", "--reverse"):
reverseOrder = 1
elif opt in ("-x", "--proxy"):
http_proxy = arg
elif opt in ("-h", "--help"):
_usage()
elif opt in ("-v", "--version"):
_version()
elif opt in ("-t", "--test"):
runTest = 1
if runTest:
setLicense(license_key)
setProxy(http_proxy)
_test()
if args and not q:
q = args[0]
func = "doGoogleSearch"
if func:
results = globals()[func]( q, http_proxy=http_proxy,
license_key=license_key )
_output(results, locals())
if __name__ == '__main__':
main(sys.argv[1:])