| Forum Details |
Topics |
Posts |
| Categories |
| |
| 576 member(s) |
| 710,006 hit(s) |
| 14 categories |
| 51 topics |
| 26 messages |
| 0 online |
|
Forums: Unfiled
| HostJ2ME.com Search Proxy |
|
|
August 26, 2009 12:36 PM
Provides search results for the OSM client by aggregating results from Google, Bing, Wikipedia, and Yahoo using the respective search APIs.
Example Usage:
SearchAgent agent = new SearchAgent(); Query query = new Query(); query.addHeader("user-agent","MySearchClient")); query.setTerm("volvo xc60"); query.setTypes(FileRecord.TYPE_ALL);
List providers = new ArrayList(); providers.add("yahoo"); providers.add("live"); providers.add("google"); providers.add("googlevideo"); providers.add("wikipedia");
BasicUIListable results = agent.find(query,5*1000,100,providers.toArray(new String[0]));
for ( SearchResult record : results.items(pn,ps) ) { System.out.println(record.getTitle()); }
Contact me if you need the full source code
|
|
|
August 26, 2009 12:40 PM
SearchAgent
/* * Copyright (C) 2005 Astrient Labs, LLC Licensed under the Apache License, * Version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. * * Astrient Labs, LLC. * www.astrientlabs.com * rashid@astrientlabs.com * Rashid Mayes 2009 */ package com.hostj2me.web.search;
import java.util.ArrayList; import java.util.List; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit;
import com.astrientlabs.biz.db.FileRecord; import com.astrientlabs.cache.Cache; import com.astrientlabs.cache.SlotCache; import com.astrientlabs.dp.task.Task; import com.astrientlabs.dp.task.TaskListener; import com.astrientlabs.dp.task.TaskRunner; import com.astrientlabs.logging.LogWriter; import com.astrientlabs.objects.ClassMap; import com.astrientlabs.objects.ClassRegister; import com.astrientlabs.webui.BasicUIListable;
public class SearchAgent implements TaskListener { private static Cache cache = new SlotCache(100); private static ClassMap classMap = new ClassMap("searchproviders"); private static TaskRunner taskRunner = new TaskRunner(8); private Semaphore semaphore = new Semaphore(0); private boolean useCaching = true; private BasicUIListable results = new BasicUIListable(); public void setUseCaching(boolean useCaching) { this.useCaching = useCaching; } public synchronized BasicUIListable find(Query query, long timeout, int maxResults, String... providers) { StringBuffer buffer = new StringBuffer(query.cacheKey()); buffer.append(timeout).append('.').append(maxResults).append('.'); for ( String provider : providers ) { buffer.append(provider).append('.'); } BasicUIListable cachedResults = null; if ( useCaching ) { cachedResults = (BasicUIListable)cache.get(buffer.toString()); } if ( cachedResults != null ) { LogWriter.system.log(getClass(), cachedResults.size() + " results found in cache."); return cachedResults; } else { semaphore.drainPermits(); Class clazz; ClassRegister register; SearchProvider searchProvider; List searchProviders = new ArrayList(providers.length); for ( String provider : providers ) { register = classMap.getRegister(provider); LogWriter.system.log(getClass(),provider + " handled by " + register); if ( register != null ) { try { clazz = register.getClassObject(); searchProvider = (SearchProvider)clazz.newInstance(); searchProvider.addListener(this); searchProvider.setQuery(query,maxResults,results); taskRunner.execute(searchProvider); searchProviders.add(searchProvider); } catch (Exception e) { LogWriter.system.log(getClass(),e); } } } try { semaphore.tryAcquire(searchProviders.size(), timeout, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { LogWriter.system.log(getClass(),e); } for ( SearchProvider p : searchProviders ) { p.cancel(); } if ( useCaching ) { cache.put(buffer.toString(),results); } return results; } } public void notify(Task job) { if ( job instanceof SearchProvider ) { SearchProvider searchProvider = (SearchProvider)job; LogWriter.system.log(getClass(), searchProvider.getName() + " completed"); semaphore.release(); } } public static void main(String[] args) { SearchAgent agent = new SearchAgent(); Query query = new Query(); query.setTerm("washington"); query.setTypes(FileRecord.TYPE_IMAGE); query.setOthers(false); query.setAdult(false); List results = agent.find(query,5*1000,50,"livevideo"); for ( SearchResult result : results ) { System.out.println(result.getTitle()); System.out.println(result.getURL()); } System.exit(0); } }
|
|
|
August 26, 2009 12:40 PM
Query:
/* * Copyright (C) 2005 Astrient Labs, LLC Licensed under the Apache License, * Version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. * * Astrient Labs, LLC. * www.astrientlabs.com * rashid@astrientlabs.com * Rashid Mayes 2009 */ package com.hostj2me.web.search;
import java.net.URLEncoder; import java.util.HashMap; import java.util.Map;
public class Query { private Map headers = new HashMap(); private String term; private int types; private boolean others; private boolean adult; public void addHeader(String key, String value) { headers.put(key, value); } protected Map getHeaderMap() { return headers; } public String cacheKey() { return new StringBuffer(term).append('.') .append(types).append('.') .append(others).append('.') .append(adult).toString(); } public boolean isAdult() { return adult; } public void setAdult(boolean adult) { this.adult = adult; } public String getTerm() { return term; } public String getTermEncoded() { return URLEncoder.encode(term); } public void setTerm(String term) { this.term = term; } public int getTypes() { return types; } public void setTypes(int types) { this.types = types; } public boolean isOthers() { return others; } public void setOthers(boolean others) { this.others = others; } }
|
|
|
August 26, 2009 12:41 PM
SearchProvider:
/* * Copyright (C) 2005 Astrient Labs, LLC Licensed under the Apache License, * Version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. * * Astrient Labs, LLC. * www.astrientlabs.com * rashid@astrientlabs.com * Rashid Mayes 2009 */ package com.hostj2me.web.search;
import java.io.UnsupportedEncodingException; import java.net.URLConnection; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.spec.InvalidKeySpecException;
import javax.crypto.BadPaddingException; import javax.crypto.IllegalBlockSizeException; import javax.crypto.NoSuchPaddingException;
import com.astrientlabs.biz.web.util.Base32; import com.astrientlabs.dp.task.Task; import com.astrientlabs.webui.BasicUIListable;
public abstract class SearchProvider extends Task { protected String name; protected Query query; protected int maxResults; protected BasicUIListable results; public String getName() { return name; } public void setQuery(Query query, int maxResults, BasicUIListable results) { this.query = query; this.maxResults = maxResults; this.results = results; } protected void prepareConnection(URLConnection connection) { for (String key : query.getHeaderMap().keySet()) { connection.setRequestProperty(key, query.getHeaderMap().get(key)); } } public void _execute() { _query(); } public static String obfuscateURL(String url) throws InvalidKeyException, IllegalBlockSizeException, BadPaddingException, InvalidAlgorithmParameterException, NoSuchPaddingException, NoSuchAlgorithmException, InvalidKeySpecException, UnsupportedEncodingException { return new String(Base32.encode(url.getBytes("UTF8"))); } public static String unobfuscateURL(String obfuscatedURL) throws InvalidKeyException, UnsupportedEncodingException, IllegalBlockSizeException, BadPaddingException, InvalidAlgorithmParameterException, NoSuchPaddingException, NoSuchAlgorithmException, InvalidKeySpecException { return new String(Base32.decode(obfuscatedURL),"UTF8"); } public abstract void _query(); public abstract void cancel(); }
|
|
|
August 26, 2009 12:42 PM
SearchResult:
/* * Copyright (C) 2005 Astrient Labs, LLC Licensed under the Apache License, * Version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. * * Astrient Labs, LLC. * www.astrientlabs.com * rashid@astrientlabs.com * Rashid Mayes 2009 */ package com.hostj2me.web.search;
public interface SearchResult { public String getTitle(); public String getMagic(); public int getMediaType(); public String getURL(); public String getId(); public long getSize(); public String getDescription(); public String getAuthor(); public SearchProvider getSearchProvider(); public String getMimeType(); public long getModified(); public int getIntID(); }
|
|
|
August 26, 2009 12:43 PM
Google:
/* * Copyright (C) 2005 Astrient Labs, LLC Licensed under the Apache License, * Version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. * * Astrient Labs, LLC. * www.astrientlabs.com * rashid@astrientlabs.com * Rashid Mayes 2009 */ package com.hostj2me.web.search;
import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection;
import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.json.XML;
import com.astrientlabs.biz.db.FileRecord; import com.astrientlabs.logging.LogWriter; import com.astrientlabs.prefs.Preferences;
public class GoogleImageSearchProvider extends SearchProvider { Preferences preferences = Preferences.getPreferences("googlesearch"); private boolean cancel; public String getName() { return "Google Image Search"; } public void _query() { int pageSize = 8; int max = Math.min(maxResults,24);//64); for (int start = 0; start max; start+=pageSize) { getResults(start); } } private void getResults(int start) { StringBuffer buffer = new StringBuffer("http://ajax.googleapis.com/ajax/services/search/images?v=1.0") .append("&appid=").append(preferences.get("key", "ENTER YOUR API KEY FROM GOOGLE")) .append("&q=").append(query.getTermEncoded()) .append("&rsz=").append("large") .append("&safe=").append(query.isAdult()?"off":"moderate") .append("&start=").append(start);
if ( cancel ) { return; } HttpURLConnection uc = null; try { URL u = new URL(buffer.toString()); uc = (HttpURLConnection) u.openConnection(); uc.addRequestProperty("Referer", "http://m.hostj2me.com"); prepareConnection(uc); String line; StringBuilder builder = new StringBuilder(); BufferedReader reader = new BufferedReader(new InputStreamReader(uc.getInputStream())); while((line = reader.readLine()) != null) { builder.append(line); } JSONObject json = new JSONObject(builder.toString()); JSONArray results = json.getJSONObject("responseData").getJSONArray("results"); GoogleSearchResult current; JSONObject result; for ( int i = 0, stop = results.length(); i stop && !cancel; i++ ) { result = results.getJSONObject(i); current = new GoogleSearchResult(); current.setSearchProvider(this); current.setMediaType(FileRecord.TYPE_IMAGE); current.setMimeType("image/jpeg"); current.setThumbnailURL(result.get("tbUrl").toString()); current.setTitle(result.get("titleNoFormatting").toString()); current.setURL(result.get("url").toString()); current.setSize(4096); this.results.add(current); } } catch (Exception e) { LogWriter.system.log(getClass(),e); } finally { try { if ( uc != null ) { uc.disconnect(); } } catch (Exception ignore){} } } public void cancel() { cancel = true; } public static void main (String[] args) { try { URL url = new URL("http://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=Paris%20Hilton"); URLConnection connection = url.openConnection(); connection.addRequestProperty("Referer", "http://www.mysite.com/index.html");
String line; StringBuilder builder = new StringBuilder(); BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); while((line = reader.readLine()) != null) { builder.append(line); }
JSONObject json = new JSONObject(builder.toString()); String xml = XML.toString(json); System.out.println(xml); } catch (MalformedURLException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (JSONException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.exit(0); } }
|
|
|
August 26, 2009 12:44 PM
Bing/MSN Live:
/* * Copyright (C) 2005 Astrient Labs, LLC Licensed under the Apache License, * Version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. * * Astrient Labs, LLC. * www.astrientlabs.com * rashid@astrientlabs.com * Rashid Mayes 2009 */ package com.hostj2me.web.search;
import java.beans.XMLDecoder; import java.net.HttpURLConnection; import java.net.URL;
import org.apache.axis.utils.XMLUtils; import org.apache.commons.lang.StringEscapeUtils;
import a.javax.swing.text.html.HTML;
import com.astrientlabs.biz.db.FileRecord; import com.astrientlabs.logging.LogWriter; import com.astrientlabs.prefs.Preferences; import com.hostj2me.xml.MiniPushHandler; import com.hostj2me.xml.MiniPushParser;
public class MSLiveSearchProvider extends SearchProvider implements MiniPushHandler { Preferences preferences = Preferences.getPreferences("livesearch"); private boolean cancel; private MiniPushParser parser = new MiniPushParser(); private MSLiveSearchResult current;
public String getName() { return "MS Live Image Search"; } public void _query() { StringBuffer buffer = new StringBuffer("http://api.search.live.net/xml.aspx?") .append("appid=").append(preferences.get("appid", "ENTER YOUR API KEY FROM BING")) .append("&sources=").append("image") .append("&query=").append(query.getTermEncoded()) .append("&image.count=").append(Math.min(maxResults,50)) .append("&adult=").append(query.isAdult()?"off":"moderate") .append("&start=").append(0);
if ( cancel ) { return; } HttpURLConnection uc = null; try { URL u = new URL(buffer.toString()); uc = (HttpURLConnection) u.openConnection(); parser.parse(uc.getInputStream(),this,1024); } catch (Exception e) { LogWriter.system.log(getClass(),e); } finally { try { if ( uc != null ) { uc.disconnect(); } } catch (Exception ignore){} } } public void handleAttribute(String tag, String attribute, String value) {
}
public void handleEndTag(String tag) { if ( tag.equalsIgnoreCase("mms:ImageResult") ) { if ( current != null ) { results.add(current); } } }
public void handleStartTag(String tag) { if ( tag.equalsIgnoreCase("mms:ImageResult") ) { current = new MSLiveSearchResult(); current.setSearchProvider(this); current.setMediaType(FileRecord.TYPE_IMAGE); current.setMimeType("image/jpeg"); } }
public void handleText(String tag, String text) { text = text.trim(); if ( tag.equalsIgnoreCase("mms:MediaUrl") ) { current.setURL(text); } if ( tag.equalsIgnoreCase("mms:Url") ) { current.setThumbnailURL(StringEscapeUtils.unescapeXml(text)); } else if ( tag.equalsIgnoreCase("mms:title") ) { current.setTitle(text); } else if ( tag.equalsIgnoreCase("mms:filesize") ) { current.setSize(Long.parseLong(text)); } else if ( tag.equalsIgnoreCase("mms:ContentType") ) { current.setMimeType(text); } } public void discard(char c) {} public void handleEndOfLineChar(char c){}; public void cancel() { cancel = true; parser.cancel(); } }
|
|
|
August 26, 2009 12:45 PM
Yahoo:
/* * Copyright (C) 2005 Astrient Labs, LLC Licensed under the Apache License, * Version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. * * Astrient Labs, LLC. * www.astrientlabs.com * rashid@astrientlabs.com * Rashid Mayes 2009 */ package com.hostj2me.web.search;
import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder;
import com.astrientlabs.biz.db.FileRecord; import com.astrientlabs.logging.LogWriter; import com.astrientlabs.prefs.Preferences; import com.hostj2me.xml.MiniPushHandler; import com.hostj2me.xml.MiniPushParser;
public class YahooSearchProvider extends SearchProvider implements MiniPushHandler { Preferences preferences = Preferences.getPreferences("yahoosearch"); private boolean cancel; private MiniPushParser parser = new MiniPushParser(); private YahooSearchResult current; private boolean thumbnail; public String getName() { return "Yahoo Image Search"; } public void _query() { StringBuffer buffer = new StringBuffer("http://search.yahooapis.com/ImageSearchService/V1/imageSearch?") .append("appid=").append(preferences.get("appid", "ENTER YOUR API KEY FROM YAHOO")) .append("&query=").append(URLEncoder.encode(query.getTerm())) .append("&results=").append(Math.min(maxResults,50)) .append("&adult_ok=").append(query.isAdult()?1:0) .append("&start=").append(0);
if ( cancel ) { return; } HttpURLConnection uc = null; try { URL u = new URL(buffer.toString()); uc = (HttpURLConnection) u.openConnection(); parser.parse(uc.getInputStream(),this,1024); } catch (Exception e) { LogWriter.system.log(getClass(),e); } finally { try { if ( uc != null ) { uc.disconnect(); } } catch (Exception ignore){} } } public void handleAttribute(String tag, String attribute, String value) {
}
public void handleEndTag(String tag) { if ( tag.equalsIgnoreCase("result") ) { if ( current != null ) { results.add(current); } } else if ( tag.equalsIgnoreCase("thumbnail") ) { thumbnail = false; } }
public void handleStartTag(String tag) { if ( tag.equalsIgnoreCase("result") ) { current = new YahooSearchResult(); current.setSearchProvider(this); current.setMediaType(FileRecord.TYPE_IMAGE); current.setMimeType("image/jpeg"); } else if ( tag.equalsIgnoreCase("thumbnail") ) { thumbnail = true; } }
public void handleText(String tag, String text) { text = text.trim(); if ( tag.equalsIgnoreCase("Url") ) { if ( thumbnail ) { current.setThumbnailURL(text); } else { current.setURL(text); } } else if ( tag.equalsIgnoreCase("title") ) { current.setTitle(text); } else if ( tag.equalsIgnoreCase("filesize") ) { current.setSize(Long.parseLong(text)); } } public void discard(char c) {} public void handleEndOfLineChar(char c){}; public void cancel() { cancel = true; parser.cancel(); } }
|
|
|
August 26, 2009 12:46 PM
Wikipedia:
/* * Copyright (C) 2005 Astrient Labs, LLC Licensed under the Apache License, * Version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. * * Astrient Labs, LLC. * www.astrientlabs.com * rashid@astrientlabs.com * Rashid Mayes 2009 */ package com.hostj2me.web.search;
import java.net.HttpURLConnection; import java.net.URL;
import com.astrientlabs.biz.db.FileRecord; import com.astrientlabs.logging.LogWriter; import com.astrientlabs.prefs.Preferences; import com.hostj2me.xml.MiniPushHandler; import com.hostj2me.xml.MiniPushParser;
public class WikipediaSearchProvider extends SearchProvider implements MiniPushHandler { Preferences preferences = Preferences.getPreferences("yahoosearch"); private boolean cancel; private MiniPushParser parser = new MiniPushParser(); private WikipediaSearchResult current; public String getName() { return "Wikipedia Search"; } public void _query() { if ( (query.getTypes() & FileRecord.TYPE_DOCUMENT) == FileRecord.TYPE_DOCUMENT ) { StringBuffer buffer = new StringBuffer("http://en.wikipedia.org/w/api.php?action=opensearch&format=xml") .append("&search=").append(query.getTermEncoded()) .append("&limit=").append(Math.min(maxResults,100));
if ( cancel ) { return; } HttpURLConnection uc = null; try { URL u = new URL(buffer.toString()); uc = (HttpURLConnection) u.openConnection(); parser.parse(uc.getInputStream(),this,1024); } catch (Exception e) { LogWriter.system.log(getClass(),e); } finally { try { if ( uc != null ) { uc.disconnect(); } } catch (Exception ignore){} } } } public void handleAttribute(String tag, String attribute, String value) {
}
public void handleEndTag(String tag) { if ( tag.equalsIgnoreCase("item") ) { if ( current != null ) { results.add(current); } } }
public void handleStartTag(String tag) { if ( tag.equalsIgnoreCase("item") ) { current = new WikipediaSearchResult(); current.setSearchProvider(this); current.setMediaType(FileRecord.TYPE_DOCUMENT); current.setMimeType("text/html"); current.setMagic("w"); current.setSize(4096); current.setThumbnailURL("https://hostj2me.com/images/mimes/48/doc.png"); } }
public void handleText(String tag, String text) { text = text.trim(); if ( tag.equalsIgnoreCase("Url") ) { current.setURL(text); } else if ( tag.equalsIgnoreCase("text") ) { current.setTitle(text); } else if ( tag.equalsIgnoreCase("description") ) { current.setDescription(text); } } public void discard(char c) {} public void handleEndOfLineChar(char c){}; public void cancel() { cancel = true; parser.cancel(); } }
|
|