Adding AppEngine files under the source control, moving downloaded code one directory deeper.

app.yaml

@@ -0,0 +1,31 @@
+application: fanfictionloader
+version: 2-5-5
+runtime: python
+api_version: 1
+
+handlers:
+- url: /generate_mock_data
+  script: mocks/generate_mock_data.py
+  login: admin
+
+- url: /r3m0v3r
+  script: utils/remover.py
+  login: admin
+
+- url: /r3m0v3r
+  script: main.py
+  login: admin
+  
+- url: /css
+  static_dir: css
+
+- url: /js
+  static_dir: js
+
+- url: /static
+  static_dir: static
+
+
+- url: /.*
+  script: main.py
+ 

cron.yaml

@@ -0,0 +1,4 @@
+cron:
+- description: cleanup job
+  url: /r3m0v3r
+  schedule: every 3 hours

css/index.css

@@ -0,0 +1,71 @@
+body
+{
+	font: 0.9em "Helvetica Neue", Arial, Helvetica, Geneva, sans-serif;
+}
+
+#main
+{
+	width: 43%;
+	margin-left: 23%;
+	background-color: #dae6ff;
+	padding: 2em;
+}
+
+#greeting
+{
+	margin-bottom: 1em;
+	border-color: #efefef;
+}
+
+
+
+#logpassword:hover, #logpasswordtable:hover, #urlbox:hover, #typebox:hover, #helpbox:hover, #yourfile:hover
+{
+	border: thin solid #fffeff;	
+}
+
+h1
+{
+	text-decoration: none;
+}
+
+#logpasswordtable
+{
+	padding: 1em;
+}
+
+#logpassword, #logpasswordtable {
+  display: none;
+}
+
+#urlbox, #typebox, #logpasswordtable, #logpassword, #helpbox, #yourfile
+{
+	margin: 1em;
+	padding: 1em;
+	border: thin dotted #fffeff;	
+}
+
+div.field
+{
+	margin-bottom: 0.5em;
+}
+
+#submitbtn
+{
+	padding: 1em;
+}
+
+#typelabel
+{
+}
+
+#typeoptions
+{
+	margin-top: 0.5em;
+}
+
+#error
+{
+	font-size: small;
+	color: #f00;
+}

delete_fic.py

@@ -0,0 +1,59 @@
+import os
+import cgi
+import sys
+import logging
+import traceback
+import StringIO
+
+from google.appengine.api import users
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp import util
+
+from fanficdownloader.downaloder import *
+from fanficdownloader.ffnet import *
+from fanficdownloader.output import *
+
+from google.appengine.ext import db
+
+from fanficdownloader.zipdir import *
+
+from ffstorage import *
+
+def create_mac(user, fic_id, fic_url):
+  return str(abs(hash(user)+hash(fic_id)))+str(abs(hash(fic_url)))
+  
+def check_mac(user, fic_id, fic_url, mac):
+  return (create_mac(user, fic_id, fic_url) == mac)
+
+def create_mac_for_fic(user, fic_id):
+  key = db.Key(fic_id)
+	fanfic = db.get(key)
+	if fanfic.user != user:
+	  return None
+	else:
+	  return create_mac(user, key, fanfic.url)
+
+class DeleteFicHandler(webapp.RequestHandler):
+	def get(self):
+		user = users.get_current_user()
+		if not user:
+			self.redirect('/login')
+
+    fic_id = self.request.get('fic_id')
+    fic_mac = self.request.get('key_id')
+    
+    actual_mac = create_mac_for_fic(user, fic_id)
+    if actual_mac != fic_mac:
+      self.response.out.write("Ooops")
+    else:
+      key = db.Key(fic_id)
+    	fanfic = db.get(key)
+      fanfic.delete()
+      self.redirect('/recent')
+    
+
+		fics = db.GqlQuery("Select * From DownloadedFanfic WHERE user = :1", user)
+		template_values = dict(fics = fics, nickname = user.nickname())
+		path = os.path.join(os.path.dirname(__file__), 'recent.html')
+		self.response.out.write(template.render(path, template_values))
+	

fanficdownloader/ffnet.py

@@ -276,9 +276,10 @@ class FFNet(FanfictionSiteAdapter):
 		return urls
 	
 	def getText(self, url):
-		time.sleep( 2.0 )
+		# time.sleep( 2.0 )
 		data = ''
 		try:
+			logging.debug("Fetching URL: %s" % url)
 			data = self.fetchUrl(url)
 		except Exception, e:
 			data = ''
@@ -301,10 +302,12 @@ class FFNet(FanfictionSiteAdapter):
 		try:
 			soup = bs.BeautifulStoneSoup(data)
 		except:
+			logging.debug(data)
 			raise FailedToDownload("Error downloading Chapter: %s!  Problem decoding page!" % url)
 
 		div = soup.find('div', {'id' : 'storytext'})
 		if None == div:
+			logging.debug(data)
 			raise FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 			
 		return div.__str__('utf8')

ffstorage.py

@@ -0,0 +1,21 @@
+from google.appengine.ext import db
+
+class OneDownload(db.Model):
+	user = db.UserProperty()
+	url = db.StringProperty()
+	format = db.StringProperty()
+	login = db.StringProperty()
+	password = db.StringProperty()
+	failure = db.StringProperty()
+	date = db.DateTimeProperty(auto_now_add=True)
+	
+class DownloadedFanfic(db.Model):
+	user = db.UserProperty()
+	url = db.StringProperty()
+	name = db.StringProperty()
+	author = db.StringProperty()
+	format = db.StringProperty()
+	date = db.DateTimeProperty(auto_now_add=True)
+	blob = db.BlobProperty()
+	mac = db.StringProperty()
+	cleared = db.BooleanProperty(default=False)

index-ajax.html

@@ -0,0 +1,109 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+	<head>
+		<link href="css/index.css" rel="stylesheet" type="text/css">
+		<link type="text/css" href="http://jqueryui.com/latest/themes/base/ui.all.css" rel="stylesheet" />
+	 
+		<title>Fanfiction Downloader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+		<script src="/js/jquery-1.3.2.js"></script>
+		<script src="/js/fdownloader.js"></script>
+		
+		<script type="text/javascript" src="http://jqueryui.com/latest/ui/ui.core.js"></script>
+	    <script type="text/javascript" src="http://jqueryui.com/latest/ui/ui.progressbar.js"></script>
+	  
+	</head>
+	<body>
+		<div id='main'>
+			<h1>
+				<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+			</h1>
+				
+<!--			<form action="/fdown" method="post"> -->
+				<div id='urlbox'>
+					<div id='greeting'>
+						Hi, {{ nickname }}! This is a fan fiction downloader, which makes reading stories from various websites much easier. Please paste a URL of the <em>first chapter</em> in the box to start. Alternatively, see your personal list of <a href="/recent">previously downloaded fanfics</a>.
+					</div>
+					<input type="text" id='url' name="url" size="50" value='{{ url }}'>
+					<div style="margin-top: 0.5em;">
+					Ebook format &nbsp;<select name="format" id="format">
+												<option value='epub'>ePub</option>
+												<option value='html'>HTML</option>
+											</select>
+					</div>
+					
+					<div id='error' style='color: red'>
+
+					</div>
+					
+					
+					
+				</div>
+				
+				<div id='yourfile' style='display:none'>
+				</div>
+				
+				<div id='typebox'>
+				</div>
+				
+				<h3>
+					Login and Password
+				</h3>
+				<div id='logpassword'>
+					If the story requires a login and password to download (e.g. marked as Mature on FFA), you may need to provide your credentials to download it, otherwise just leave it empty
+				</div>
+				<div id='logpasswordtable'>
+					<div class='fieldandlabel'>
+						<div class='label'>Login</div>
+						<div class='field'><input type='text' name='login' id='login' size='50'></div>
+					</div>
+					
+					<div class='fieldandlabel'>
+						<div class='label'>Password</div>
+						<div class='field'><input type='password' id='password' name='password' size='50'></div>
+					</div>
+				</div>
+				<div id='submitbtn'>
+					<span id='submit_button'><button onclick='downloadFanfic();'>Download</button></span>
+					<span id='ajax_loader' style='display:none'><img src="/static/ajax-loader.gif"></span>
+				</div>
+
+				
+				<div id="progressbar">
+					
+				</div>
+				<div id='helpbox'>
+				Few things to know, which will make your life substantially easier:
+				<ol>
+					<li>Small <a href="http://www.sigizmund.com/reading-fanfiction-off-line-in-stanza-and-oth">post written by me</a> &mdash; how to read fiction in Stanza or any other ebook reader. </a></li>
+					<li>Currently we support fanfiction.net, fictionpress.com, fanficauthors.net and ficwad.com</li>
+					<li>Paste a URL of the first chapter of the fanfic, not the index page</li>
+					<li>Fics with a single chapter are not supported (you can just copy and paste it)</li>
+					<li>Stories which are too long may not be downloaded correctly and application will report a time-out error &mdash; this is a limitation which is currently imposed by Google AppEngine on a long-running activities</li>
+					<li>FicWad support is somewhat flaky &mdash; if you feel it doesn't work for you, send all the details to me</li>
+					<li>You can download fanfics and store them for 'later' by just downloading them and visiting <a href="/recent">recent downloads</a> section, but in future they will be deleted after 5 days to save the space</li>
+					<li>If Downloader simply opens a download file window rather than saves the fanfic and gives you a link, it means it is too large to save in the database and you need to download it straight away</li>
+					<li>If you think that something that should work in fact doesn't, drop me a mail to <a href='mailto:sigizmund@gmail.com'>sigizmund@gmail.com</a></li>
+				</ol>
+				Otherwise, just have fun, and if you want to say thank you &mdash; use the email above.
+				</div>
+				<div style='text-align: center'>
+					<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
+					alt="Powered by Google App Engine" />
+					<br/><br/>
+					FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+					Copyright &copy; <a href="http://twitter.com/sigizmund">Roman Kirillov</a>
+				</div>
+<!--			</form> -->
+		</div>
+		<script type="text/javascript">
+		var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+		document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+		</script>
+		<script type="text/javascript">
+		try {
+		var pageTracker = _gat._getTracker("UA-12136939-1");
+		pageTracker._trackPageview();
+		} catch(err) {}</script>
+	</body>
+</html>

index.html

@@ -0,0 +1,189 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+	<head>
+		<link href="css/index.css" rel="stylesheet" type="text/css">
+		<title>Fanfiction Downloader &mdash; twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com to epub and HTML to Stanza, Kindle, Nook, Sony Reader</title>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+		<meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
+	</head>
+	<body>
+		<div id='main'>
+			<h1>
+				<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+			</h1>
+			
+			<div style="text-align: center">
+                <script type="text/javascript"><!--
+                google_ad_client = "pub-2027714004231956";
+                /* FFD */
+                google_ad_slot = "7330682770";
+                google_ad_width = 468;
+                google_ad_height = 60;
+                //-->
+                </script>
+                <script type="text/javascript"
+                src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+                </script>
+            </div>
+<!--			<div id='yourfile'> -->
+			{{yourfile}}
+<!--			</div> -->
+
+			{% if authorized %}
+			<form action="/fdown" method="post">
+				<div id='urlbox'>
+					<div id='greeting'>
+					  Hi, {{ nickname }}! This is a fan fiction downloader, which makes reading stories from various websites
+					  much easier. 
+					  <ul>
+					    <li>
+					      For fictionalley.org, please paste the URL of the story's chapter list in the box, such as
+					      <a href="http://www.fictionalley.org/authors/drt/DA.html">this</a>.  Or the story text URL for
+					      fictionalley.org one-shots, such
+					      as <a href="http://www.fictionalley.org/authors/drt/JOTP01a.html">this</a>.
+					    </li>
+					    <li>
+					      For all other supported sites, please paste the URL of the <em>first chapter</em> in the box.  For
+					      one-shots, the first chapter is the whole story.
+					    </li>
+					    <li>
+					      Alternatively, see your personal list of <a href="/recent">previously downloaded fanfics</a>.
+					    </li>
+					  </ul>	
+					</div>
+					<div id='error'>
+					{{ error_message }}
+					</div>
+					<input type="text" name="url" size="50" value='{{ url }}'>
+				</div>
+				
+				<div id='typebox'>
+					<div id='typelabel'>Ebook format</div>
+					<div id='typeoptions'>
+						<input type='radio' name='format' value='epub' checked>EPub</input>
+						<input type='radio' name='format' value='html'>HTML</input>
+						<input type='radio' name='format' value='text'>Plain Text</input>
+					</div>
+				</div>
+				
+				<h3>Login and Password</h3>
+				<div id='logpassword'>
+				  
+					If the story requires a login and password to download (e.g. marked as Mature on FFA), you may need to provide
+					your credentials to download it, otherwise just leave it empty
+				</div>
+				<div id='logpasswordtable'>
+					<div class='fieldandlabel'>
+						<div class='label'>Login</div>
+						<div class='field'><input type='text' name='login' size='50'></div>
+					</div>
+					
+					<div class='fieldandlabel'>
+						<div class='label'>Password</div>
+						<div class='field'><input type='password' name='password' size='50'></div>
+					</div>
+				</div>
+				<div id='submitbtn'>
+					<input type="submit" value="Download">
+				</div>
+			</form>
+			{% else %}
+			<div id='urlbox'>
+				<div id='greeting'>
+					<p>
+					  This is a fan fiction downloader, which makes reading stories from various websites much easier. Before you
+					  can start downloading fanfics, you need to login, so downloader can remember your fanfics and store them.
+					</p>
+					<p><a href="{{ login_url }}">Login using Google account</a></p>
+				</div>
+			</div>
+			{% endif %}
+			
+				<div id='helpbox'>
+				Few things to know, which will make your life substantially easier:
+				<ol>
+				  <li>
+				    First thing to know: I do not use your login and password. In fact, all I know about it is your ID &ndash; password
+				    is being verified by Google and is absolutely, totally unknown to anyone but you.
+				  </li>
+				  <li>
+				    Small <a href="http://www.sigizmund.com/reading-fanfiction-off-line-in-stanza-and-oth">post written by me</a>
+				    &mdash; how to read fiction in Stanza or any other ebook reader.
+				  </li>
+				  <li>
+				    Currently we support fanfiction.net, fictionpress.com, ficwad.com, fictionalley.org, harrypotterfanfiction.com, potionsandsnitches.net, and twilighted.net.
+				    (fanficauthors.net withdrawn as they offer native ePub functionality now.)
+				  </li>
+				  <li>
+				    You can download fanfiction directly from your iPhone, Kindle or (possibly) other ebook reader.
+				  </li>
+				  <li>
+				    Paste a URL of the first chapter of the fanfic, not the index page, except for fictionalley.org.
+				  </li>
+				  <li>
+				    For fictionalley.org, you need to use the URL of the story's chapter list, such as
+				    <a href="http://www.fictionalley.org/authors/drt/DA.html">this</a>.  Or the story text URL for fictionalley.org
+				    one-shots, such as <a href="http://www.fictionalley.org/authors/drt/JOTP01a.html">this</a>.
+				  </li>
+				  <li>
+				    One-shots, fics with a single chapter, <em>are</em> now supported.
+				  </li>
+				  <li>
+				    You can download fanfics and store them for 'later' by just downloading them and visiting <a href="/recent">recent
+				    downloads</a> section.
+				  </li>
+				  <li>
+				    Downloaded stories are deleted after some time (which should give you enough of time to download it and will keep
+				    Google happy about the app not going over the storage limit).
+				  </li>
+				  <li>
+				    If Downloader simply opens a download file window rather than saves the fanfic and gives you a link, it means it is
+				    too large to save in the database and you need to download it straight away.
+				  </li>
+				  <li>
+				    If you see some funny characters in downloaded Plain Text file, make sure you choose text file encoding UTF-8 and
+				    not something else.
+				  </li>
+				  <li>
+				    If you think that something that should work in fact doesn't, drop me a mail
+				    to <a href='mailto:sigizmund@gmail.com'>sigizmund@gmail.com</a>, or, even better, write an email to
+				    our <a href="http://groups.google.com/group/fanfic-downloader">Google Group</a>. I also encourage you to join it so
+				    you will find out about latest updates and fixes as soon as possible
+				  </li>
+				</ol>
+				Otherwise, just have fun, and if you want to say thank you &mdash; use the contacts above. 
+				</div>
+				<div style='text-align: center'>
+					<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
+					alt="Powered by Google App Engine" />
+					<br/><br/>
+					FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+					Copyright &copy; <a href="http://twitter.com/sigizmund">Roman Kirillov</a>
+				</div>
+
+			<div style="margin-top: 1em; text-align: center'">
+                <script type="text/javascript"><!--
+                google_ad_client = "pub-2027714004231956";
+                /* FFD */
+                google_ad_slot = "7330682770";
+                google_ad_width = 468;
+                google_ad_height = 60;
+                //-->
+                </script>
+                <script type="text/javascript"
+                src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+                </script>
+            </div>
+			
+		</div>
+		<script type="text/javascript">
+		var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+		document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+		</script>
+		<script type="text/javascript">
+		try {
+		var pageTracker = _gat._getTracker("UA-12136939-1");
+		pageTracker._trackPageview();
+		} catch(err) {}</script>
+	</body>
+</html>

index.yaml

@@ -0,0 +1,22 @@
+indexes:
+
+# AUTOGENERATED
+
+# This index.yaml is automatically updated whenever the dev_appserver
+# detects that a new type of query is run.  If you want to manage the
+# index.yaml file manually, remove the above marker line (the line
+# saying "# AUTOGENERATED").  If you want to manage some indexes
+# manually, move them above the marker line.  The index.yaml file is
+# automatically uploaded to the admin console when you next deploy
+# your application using appcfg.py.
+
+- kind: DownloadedFanfic
+  properties:
+  - name: cleared
+  - name: date
+
+- kind: DownloadedFanfic
+  properties:
+  - name: user
+  - name: date
+    direction: desc

js/fdownloader.js

@@ -0,0 +1,116 @@
+var g_CurrentKey = null;
+var g_Counter = 0;
+
+var COUNTER_MAX = 50;
+
+
+function setErrorState(error)
+{
+	olderr = error;
+	error = error + "<br/><a href='mailto:sigizmund@gmail.com?subject=Problem with the fanfiction downloader'>" + "Complain about this error</a>";
+	$('#error').html(error);
+}
+
+function clearErrorState()
+{
+	$('#error').html('');
+}
+
+function showFile(data)
+{
+	$('#yourfile').html('<a href="/file?id=' + data.key + '">' + data.name + " by " + data.author + "</a>");
+	$('#yourfile').show();
+}
+
+function hideFile()
+{
+	$('#yourfile').hide();
+}
+
+function checkResults()
+{
+	if ( g_Counter >= COUNTER_MAX )
+	{
+		return;
+	}
+	
+	g_Counter+=1;
+
+	$.getJSON('/progress', { 'key' : g_CurrentKey }, function(data)
+	{
+		if ( data.result != "Nope")
+		{
+			if ( data.result != "OK" )
+			{
+				leaveLoadingState();
+				setErrorState(data.result);
+			}
+			else
+			{
+				showFile(data);
+				leaveLoadingState();
+				// result = data.split("|");
+				// showFile(result[1], result[2], result[3]);
+			}
+			
+			$("#progressbar").progressbar('destroy');
+			g_Counter = 101;
+		}
+	});
+	
+	if ( g_Counter < COUNTER_MAX ) 
+		setTimeout("checkResults()", 1000);
+	else
+	{
+		leaveLoadingState();
+		setErrorState("Operation takes too long - terminating by timeout (story too long?)");
+	}
+}
+
+function enterLoadingState()
+{
+	$('#submit_button').hide();
+	$('#ajax_loader').show();
+}
+
+function leaveLoadingState()
+{
+	$('#submit_button').show();
+	$('#ajax_loader').hide();
+}
+
+function downloadFanfic()
+{
+	clearErrorState();
+	hideFile();
+
+
+	format = $("#format").val();
+	alert(format);
+	
+	return;
+	
+	var url = $('#url').val();
+	var login = $('#login').val();
+	var password = $('#password').val();
+	
+	if ( url == '' )
+	{
+		setErrorState('URL shouldn\'t be empty');
+		return;
+	}
+	
+	if ( (url.indexOf('fanfiction.net') == -1 && url.indexOf('fanficauthors') == -1 && url.indexOf('ficwad') == -1  &&  url.indexOf('fictionpress') == -1) || (url.indexOf('adultfanfiction.net') != -1) )
+	{
+		setErrorState("This source is not yet supported. Ping me if you want it!");
+		return;
+	}
+	
+	$.post('/submitDownload', {'url' : url, 'login' : login, 'password' : password, 'format' : format}, function(data)
+	{
+		g_CurrentKey = data;
+		g_Counter = 0;
+		setTimeout("checkResults()", 1000);
+		enterLoadingState();
+	})
+}

main.py

@@ -0,0 +1,309 @@
+#!/usr/bin/env python
+#
+# Copyright 2007 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import logging
+import traceback
+import StringIO
+
+from google.appengine.runtime import DeadlineExceededError
+
+from google.appengine.ext.webapp import template
+from google.appengine.api import users
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp import util
+
+from fanficdownloader.downloader import *
+from fanficdownloader.ffnet import *
+from fanficdownloader.output import *
+from fanficdownloader import twilighted
+
+from google.appengine.ext import db
+
+from fanficdownloader.zipdir import *
+
+from ffstorage import *
+
+
+
+class LoginRequired(webapp.RequestHandler):
+	def get(self):
+		user = users.get_current_user()
+		if user:
+			self.redirect('/')
+		else:
+			logging.debug(users.create_login_url('/'))
+			url = users.create_login_url(self.request.uri)
+			template_values = {'login_url' : url}
+			path = os.path.join(os.path.dirname(__file__), 'index-nonlogin.html')
+			self.response.out.write(template.render(path, template_values))
+
+class MainHandler(webapp.RequestHandler):
+	def get(self):
+		user = users.get_current_user()
+		if user:
+			error = self.request.get('error')
+			template_values = {'nickname' : user.nickname(), 'authorized': True}
+			url = self.request.get('url')
+			template_values['url'] = url
+			
+			if error != None and len(error) > 1:
+				if error == 'login_required':
+					template_values['error_message'] = 'This story (or one of the chapters) requires you to be logged in.'
+				elif error == 'bad_url':
+					template_values['error_message'] = 'Unsupported URL: ' + url
+				elif error == 'custom':
+					template_values['error_message'] = 'Error happened: ' + self.request.get('errtext')
+			
+			filename = self.request.get('file')
+			if len(filename) > 1:
+				template_values['yourfile'] = '''<div id='yourfile'><a href='/file?id=%s'>"%s" by %s</a></div>''' % (filename, self.request.get('name'), self.request.get('author'))
+			
+			self.response.headers['Content-Type'] = 'text/html'
+			path = os.path.join(os.path.dirname(__file__), 'index.html')
+
+			self.response.out.write(template.render(path, template_values))
+		else:
+#			self.redirect(users.create_login_url(self.request.uri))
+#			self.redirect('/login')
+			logging.debug(users.create_login_url('/'))
+			url = users.create_login_url(self.request.uri)
+			template_values = {'login_url' : url, 'authorized': False}
+			path = os.path.join(os.path.dirname(__file__), 'index.html')
+			self.response.out.write(template.render(path, template_values))
+
+
+class FileServer(webapp.RequestHandler):
+	def get(self):
+#		user  = users.get_current_user()
+		fileId = self.request.get('id')
+		
+		if fileId == None or len(fileId) < 3:
+			self.redirect('/')
+		
+		key = db.Key(fileId)
+		fanfic = db.get(key)
+		
+		name = fanfic.name.encode('utf-8')
+		
+		name = makeAcceptableFilename(name)
+		
+		logging.info("Serving file: %s" % name)
+
+		if fanfic.format == 'epub':
+			self.response.headers['Content-Type'] = 'application/epub+zip'
+			self.response.headers['Content-disposition'] = 'attachment; filename=' + name + '.epub'
+		elif fanfic.format == 'html':
+			self.response.headers['Content-Type'] = 'text/html'
+			self.response.headers['Content-disposition'] = 'attachment; filename=' + name + '.html.zip'
+		elif fanfic.format == 'text':
+			self.response.headers['Content-Type'] = 'text/plain'
+			self.response.headers['Content-disposition'] = 'attachment; filename=' +name + '.txt.zip'
+		
+		self.response.out.write(fanfic.blob)
+
+class RecentFilesServer(webapp.RequestHandler):
+	def get(self):
+		user = users.get_current_user()
+		if not user:
+			self.redirect('/login')
+		
+#		fics = db.GqlQuery("Select * From DownloadedFanfic WHERE user = :1 and cleared = :2", user)
+		q = DownloadedFanfic.all()
+		q.filter('user =', user)
+		q.filter('cleared =', False)
+		fics = q.fetch(100)
+		
+		template_values = dict(fics = fics, nickname = user.nickname())
+		path = os.path.join(os.path.dirname(__file__), 'recent.html')
+		self.response.out.write(template.render(path, template_values))
+		
+class RecentAllFilesServer(webapp.RequestHandler):
+	def get(self):
+		user = users.get_current_user()
+		if user.nickname() != 'sigizmund':
+			return
+			
+		fics = db.GqlQuery("Select * From DownloadedFanfic")
+		template_values = dict(fics = fics, nickname = user.nickname())
+		path = os.path.join(os.path.dirname(__file__), 'recent.html')
+		self.response.out.write(template.render(path, template_values))	
+
+class FanfictionDownloader(webapp.RequestHandler):
+	def _printableVersion(self, text):
+		text = removeEntities(text)
+		try:
+			d = text.decode('utf-8')
+		except:
+			d = text
+		return d
+	
+
+	def post(self):
+		logging.getLogger().setLevel(logging.DEBUG)
+		
+		user = users.get_current_user()
+		if not user:
+			self.redirect(users.create_login_url('/'))
+		
+		format = self.request.get('format')
+		url = self.request.get('url')
+		login = self.request.get('login')
+		password = self.request.get('password')
+		
+		logging.info("Downloading: " + url)
+		
+		adapter = None
+		writerClass = None
+
+		download = OneDownload()
+		download.user = user
+		download.url = url
+		download.login = login
+		download.password = password
+		download.format = format
+		logging.info('Creating adapter...')
+		
+		try:
+			if url.find('fictionalley') != -1:
+				adapter = fictionalley.FictionAlley(url)
+			elif url.find('ficwad') != -1:
+				adapter = ficwad.FicWad(url)
+			elif url.find('fanfiction.net') != -1 or url.find('fictionpress.com') != -1:
+				adapter = ffnet.FFNet(url)
+			elif url.find('harrypotterfanfiction.com') != -1:
+				adapter = hpfiction.HPFiction(url)
+			elif url.find('twilighted') != -1 or url.find('potionsandsnitches.net') != -1:
+				adapter = twilighted.Twilighted(url)
+			else:
+				logging.debug("Bad URL detected")
+				self.redirect('/?error=bad_url&url=' + urlEscape(url) )
+				return
+		except Exception, e:
+			logging.exception(e)
+			download.failure = "Adapter was not created: " + str(e)
+			download.put()
+			
+			self.redirect('/?error=custom&url=' + urlEscape(url) + '&errtext=' + urlEscape(str(traceback.format_exc())) )
+			return
+		
+		logging.info('Created an adaper: %s' % adapter)
+		
+		if len(login) > 1:
+			adapter.setLogin(login)
+			adapter.setPassword(password)
+
+		if format == 'epub':
+			writerClass = output.EPubFanficWriter
+		elif format == 'html':
+			writerClass = output.HTMLWriter
+		else:
+			writerClass = output.TextWriter
+		
+		loader = FanficLoader(adapter, writerClass, quiet = True, inmemory=True, compress=False)
+		try:
+			data = loader.download()
+			
+			if format == 'html' or format == 'text':
+				# data is uncompressed hence huge
+				ext = '.html'
+				if format == 'text':
+					ext = '.txt'
+				files = {makeAcceptableFilename(str(adapter.getStoryName())) + ext : StringIO.StringIO(data.decode('utf-8')) }
+				d = inMemoryZip(files)
+				data = d.getvalue()
+			
+		
+		except LoginRequiredException, e:
+			logging.exception(e)
+			download.failure = 'Login problem detected'
+			download.put()
+			
+			self.redirect('/?error=login_required&url=' + urlEscape(url))
+			return
+		except:
+			e = sys.exc_info()[0]
+			
+			logging.exception(e)
+			download.failure = 'Some exception happened in downloader: ' + str(e)
+			download.put()
+			
+			self.redirect('/?error=custom&url=' + urlEscape(url) + '&errtext=' + urlEscape(str(traceback.format_exc())) )
+			return
+			
+		if data == None:
+			if loader.badLogin:
+				logging.debug("Bad login detected")
+				
+				download.failure = 'Login problem detected'
+				download.put()
+				
+				self.redirect('/?error=login_required&url=' + urlEscape(url))
+		else:
+			fic = DownloadedFanfic()
+			fic.user = user
+			fic.url = url
+			fic.format = format
+			fic.name = self._printableVersion(adapter.getStoryName())
+			fic.author = self._printableVersion(adapter.getAuthorName())
+			fic.blob = data
+			
+			try:
+				fic.put()
+				
+				key = fic.key()
+				
+				download.put()
+				self.redirect('/?file='+str(key)+'&name=' + urlEscape(fic.name) + '&author=' + urlEscape(fic.author))
+				
+				logging.info("Download finished OK")
+			except Exception, e:
+				logging.exception(e)
+				# it was too large, won't save it
+				name = str(makeAcceptableFilename(adapter.getStoryName()))
+				if format == 'epub':
+					self.response.headers['Content-Type'] = 'application/epub+zip'
+					self.response.headers['Content-disposition'] = 'attachment; filename=' + name + '.epub'
+				elif format == 'html':
+					self.response.headers['Content-Type'] = 'application/zip'
+					self.response.headers['Content-disposition'] = 'attachment; filename=' + name + '.html.zip'
+				elif format == 'text':
+					self.response.headers['Content-Type'] = 'application/zip'
+					self.response.headers['Content-disposition'] = 'attachment; filename=' + name + '.txt.zip'
+				
+				self.response.out.write(data)
+				
+def toPercentDecimal(match): 
+	"Return the %decimal number for the character for url escaping"
+	s = match.group(1)
+	return "%%%02x" % ord(s)
+
+def urlEscape(data):
+	"Escape text, including unicode, for use in URLs"
+	p = re.compile(r'([^\w])')
+	return p.sub(toPercentDecimal, data.encode("utf-8"))
+
+def main():
+  application = webapp.WSGIApplication([('/', MainHandler), ('/fdown', FanfictionDownloader), ('/file', FileServer), ('/recent', RecentFilesServer), ('/r2d2', RecentAllFilesServer), ('/login', LoginRequired)],
+                                       debug=False)
+  util.run_wsgi_app(application)
+
+
+if __name__ == '__main__':
+	logging.getLogger().setLevel(logging.DEBUG)
+	main()

queue.yaml

@@ -0,0 +1,5 @@
+queue:
+- name: default
+  rate: 1/s
+- name: download
+  rate: 10/s

recent.html

@@ -0,0 +1,69 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+	<head>
+		<link href="css/index.css" rel="stylesheet" type="text/css">
+		<title>Fanfiction Downloader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+	</head>
+	<body>
+		<div id='main'>
+			<h1>
+				<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+			</h1>
+			
+			
+			<script type="text/javascript"><!--
+			google_ad_client = "pub-2027714004231956";
+			/* 468x60, created 6/9/10 */
+			google_ad_slot = "8817097473";
+			google_ad_width = 468;
+			google_ad_height = 60;
+			//-->
+			</script>
+			<script type="text/javascript"
+			src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+			</script>			
+			
+			
+			
+<!--			<div id='yourfile'> -->
+			{{yourfile}}
+<!--			</div> -->
+			
+				<div id='urlbox'>
+					<div id='greeting'>
+						Hi, {{ nickname }}! These fanfics you've downloaded previously.
+					</div>
+				</div>
+				
+				<div id='helpbox'>
+					{% for fic in fics %}
+					<p> <a href="/file?id={{ fic.key }}">{{ fic.name }}</a> by {{ fic.author }} ({{ fic.format }})<br/><small><a href="{{ fic.url }}">{{ fic.url }}</a></small></p>
+					{% endfor %}
+				</div>
+				
+				<script type="text/javascript"><!--
+				google_ad_client = "pub-2027714004231956";
+				/* 468x60, created 6/9/10 */
+				google_ad_slot = "2009456648";
+				google_ad_width = 468;
+				google_ad_height = 60;
+				//-->
+				</script>
+				<script type="text/javascript"
+				src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+				</script>
+				
+				
+		</div>
+		<script type="text/javascript">
+		var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+		document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+		</script>
+		<script type="text/javascript">
+		try {
+		var pageTracker = _gat._getTracker("UA-12136939-1");
+		pageTracker._trackPageview();
+		} catch(err) {}</script>
+	</body>
+</html>

simplejson/__init__.py

@@ -0,0 +1,318 @@
+r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of
+JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
+interchange format.
+
+:mod:`simplejson` exposes an API familiar to users of the standard library
+:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained
+version of the :mod:`json` library contained in Python 2.6, but maintains
+compatibility with Python 2.4 and Python 2.5 and (currently) has
+significant performance advantages, even without using the optional C
+extension for speedups.
+
+Encoding basic Python object hierarchies::
+
+    >>> import simplejson as json
+    >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
+    '["foo", {"bar": ["baz", null, 1.0, 2]}]'
+    >>> print json.dumps("\"foo\bar")
+    "\"foo\bar"
+    >>> print json.dumps(u'\u1234')
+    "\u1234"
+    >>> print json.dumps('\\')
+    "\\"
+    >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)
+    {"a": 0, "b": 0, "c": 0}
+    >>> from StringIO import StringIO
+    >>> io = StringIO()
+    >>> json.dump(['streaming API'], io)
+    >>> io.getvalue()
+    '["streaming API"]'
+
+Compact encoding::
+
+    >>> import simplejson as json
+    >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':'))
+    '[1,2,3,{"4":5,"6":7}]'
+
+Pretty printing::
+
+    >>> import simplejson as json
+    >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)
+    >>> print '\n'.join([l.rstrip() for l in  s.splitlines()])
+    {
+        "4": 5,
+        "6": 7
+    }
+
+Decoding JSON::
+
+    >>> import simplejson as json
+    >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}]
+    >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj
+    True
+    >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar'
+    True
+    >>> from StringIO import StringIO
+    >>> io = StringIO('["streaming API"]')
+    >>> json.load(io)[0] == 'streaming API'
+    True
+
+Specializing JSON object decoding::
+
+    >>> import simplejson as json
+    >>> def as_complex(dct):
+    ...     if '__complex__' in dct:
+    ...         return complex(dct['real'], dct['imag'])
+    ...     return dct
+    ...
+    >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}',
+    ...     object_hook=as_complex)
+    (1+2j)
+    >>> import decimal
+    >>> json.loads('1.1', parse_float=decimal.Decimal) == decimal.Decimal('1.1')
+    True
+
+Specializing JSON object encoding::
+
+    >>> import simplejson as json
+    >>> def encode_complex(obj):
+    ...     if isinstance(obj, complex):
+    ...         return [obj.real, obj.imag]
+    ...     raise TypeError(repr(o) + " is not JSON serializable")
+    ...
+    >>> json.dumps(2 + 1j, default=encode_complex)
+    '[2.0, 1.0]'
+    >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j)
+    '[2.0, 1.0]'
+    >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j))
+    '[2.0, 1.0]'
+
+
+Using simplejson.tool from the shell to validate and pretty-print::
+
+    $ echo '{"json":"obj"}' | python -m simplejson.tool
+    {
+        "json": "obj"
+    }
+    $ echo '{ 1.2:3.4}' | python -m simplejson.tool
+    Expecting property name: line 1 column 2 (char 2)
+"""
+__version__ = '2.0.9'
+__all__ = [
+    'dump', 'dumps', 'load', 'loads',
+    'JSONDecoder', 'JSONEncoder',
+]
+
+__author__ = 'Bob Ippolito <bob@redivi.com>'
+
+from decoder import JSONDecoder
+from encoder import JSONEncoder
+
+_default_encoder = JSONEncoder(
+    skipkeys=False,
+    ensure_ascii=True,
+    check_circular=True,
+    allow_nan=True,
+    indent=None,
+    separators=None,
+    encoding='utf-8',
+    default=None,
+)
+
+def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
+        allow_nan=True, cls=None, indent=None, separators=None,
+        encoding='utf-8', default=None, **kw):
+    """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
+    ``.write()``-supporting file-like object).
+
+    If ``skipkeys`` is true then ``dict`` keys that are not basic types
+    (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
+    will be skipped instead of raising a ``TypeError``.
+
+    If ``ensure_ascii`` is false, then the some chunks written to ``fp``
+    may be ``unicode`` instances, subject to normal Python ``str`` to
+    ``unicode`` coercion rules. Unless ``fp.write()`` explicitly
+    understands ``unicode`` (as in ``codecs.getwriter()``) this is likely
+    to cause an error.
+
+    If ``check_circular`` is false, then the circular reference check
+    for container types will be skipped and a circular reference will
+    result in an ``OverflowError`` (or worse).
+
+    If ``allow_nan`` is false, then it will be a ``ValueError`` to
+    serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
+    in strict compliance of the JSON specification, instead of using the
+    JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
+
+    If ``indent`` is a non-negative integer, then JSON array elements and object
+    members will be pretty-printed with that indent level. An indent level
+    of 0 will only insert newlines. ``None`` is the most compact representation.
+
+    If ``separators`` is an ``(item_separator, dict_separator)`` tuple
+    then it will be used instead of the default ``(', ', ': ')`` separators.
+    ``(',', ':')`` is the most compact JSON representation.
+
+    ``encoding`` is the character encoding for str instances, default is UTF-8.
+
+    ``default(obj)`` is a function that should return a serializable version
+    of obj or raise TypeError. The default simply raises TypeError.
+
+    To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
+    ``.default()`` method to serialize additional types), specify it with
+    the ``cls`` kwarg.
+
+    """
+    # cached encoder
+    if (not skipkeys and ensure_ascii and
+        check_circular and allow_nan and
+        cls is None and indent is None and separators is None and
+        encoding == 'utf-8' and default is None and not kw):
+        iterable = _default_encoder.iterencode(obj)
+    else:
+        if cls is None:
+            cls = JSONEncoder
+        iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
+            check_circular=check_circular, allow_nan=allow_nan, indent=indent,
+            separators=separators, encoding=encoding,
+            default=default, **kw).iterencode(obj)
+    # could accelerate with writelines in some versions of Python, at
+    # a debuggability cost
+    for chunk in iterable:
+        fp.write(chunk)
+
+
+def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
+        allow_nan=True, cls=None, indent=None, separators=None,
+        encoding='utf-8', default=None, **kw):
+    """Serialize ``obj`` to a JSON formatted ``str``.
+
+    If ``skipkeys`` is false then ``dict`` keys that are not basic types
+    (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
+    will be skipped instead of raising a ``TypeError``.
+
+    If ``ensure_ascii`` is false, then the return value will be a
+    ``unicode`` instance subject to normal Python ``str`` to ``unicode``
+    coercion rules instead of being escaped to an ASCII ``str``.
+
+    If ``check_circular`` is false, then the circular reference check
+    for container types will be skipped and a circular reference will
+    result in an ``OverflowError`` (or worse).
+
+    If ``allow_nan`` is false, then it will be a ``ValueError`` to
+    serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
+    strict compliance of the JSON specification, instead of using the
+    JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
+
+    If ``indent`` is a non-negative integer, then JSON array elements and
+    object members will be pretty-printed with that indent level. An indent
+    level of 0 will only insert newlines. ``None`` is the most compact
+    representation.
+
+    If ``separators`` is an ``(item_separator, dict_separator)`` tuple
+    then it will be used instead of the default ``(', ', ': ')`` separators.
+    ``(',', ':')`` is the most compact JSON representation.
+
+    ``encoding`` is the character encoding for str instances, default is UTF-8.
+
+    ``default(obj)`` is a function that should return a serializable version
+    of obj or raise TypeError. The default simply raises TypeError.
+
+    To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
+    ``.default()`` method to serialize additional types), specify it with
+    the ``cls`` kwarg.
+
+    """
+    # cached encoder
+    if (not skipkeys and ensure_ascii and
+        check_circular and allow_nan and
+        cls is None and indent is None and separators is None and
+        encoding == 'utf-8' and default is None and not kw):
+        return _default_encoder.encode(obj)
+    if cls is None:
+        cls = JSONEncoder
+    return cls(
+        skipkeys=skipkeys, ensure_ascii=ensure_ascii,
+        check_circular=check_circular, allow_nan=allow_nan, indent=indent,
+        separators=separators, encoding=encoding, default=default,
+        **kw).encode(obj)
+
+
+_default_decoder = JSONDecoder(encoding=None, object_hook=None)
+
+
+def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None,
+        parse_int=None, parse_constant=None, **kw):
+    """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
+    a JSON document) to a Python object.
+
+    If the contents of ``fp`` is encoded with an ASCII based encoding other
+    than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must
+    be specified. Encodings that are not ASCII based (such as UCS-2) are
+    not allowed, and should be wrapped with
+    ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode``
+    object and passed to ``loads()``
+
+    ``object_hook`` is an optional function that will be called with the
+    result of any object literal decode (a ``dict``). The return value of
+    ``object_hook`` will be used instead of the ``dict``. This feature
+    can be used to implement custom decoders (e.g. JSON-RPC class hinting).
+
+    To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
+    kwarg.
+
+    """
+    return loads(fp.read(),
+        encoding=encoding, cls=cls, object_hook=object_hook,
+        parse_float=parse_float, parse_int=parse_int,
+        parse_constant=parse_constant, **kw)
+
+
+def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None,
+        parse_int=None, parse_constant=None, **kw):
+    """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON
+    document) to a Python object.
+
+    If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding
+    other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name
+    must be specified. Encodings that are not ASCII based (such as UCS-2)
+    are not allowed and should be decoded to ``unicode`` first.
+
+    ``object_hook`` is an optional function that will be called with the
+    result of any object literal decode (a ``dict``). The return value of
+    ``object_hook`` will be used instead of the ``dict``. This feature
+    can be used to implement custom decoders (e.g. JSON-RPC class hinting).
+
+    ``parse_float``, if specified, will be called with the string
+    of every JSON float to be decoded. By default this is equivalent to
+    float(num_str). This can be used to use another datatype or parser
+    for JSON floats (e.g. decimal.Decimal).
+
+    ``parse_int``, if specified, will be called with the string
+    of every JSON int to be decoded. By default this is equivalent to
+    int(num_str). This can be used to use another datatype or parser
+    for JSON integers (e.g. float).
+
+    ``parse_constant``, if specified, will be called with one of the
+    following strings: -Infinity, Infinity, NaN, null, true, false.
+    This can be used to raise an exception if invalid JSON numbers
+    are encountered.
+
+    To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
+    kwarg.
+
+    """
+    if (cls is None and encoding is None and object_hook is None and
+            parse_int is None and parse_float is None and
+            parse_constant is None and not kw):
+        return _default_decoder.decode(s)
+    if cls is None:
+        cls = JSONDecoder
+    if object_hook is not None:
+        kw['object_hook'] = object_hook
+    if parse_float is not None:
+        kw['parse_float'] = parse_float
+    if parse_int is not None:
+        kw['parse_int'] = parse_int
+    if parse_constant is not None:
+        kw['parse_constant'] = parse_constant
+    return cls(encoding=encoding, **kw).decode(s)

simplejson/decoder.py

@@ -0,0 +1,354 @@
+"""Implementation of JSONDecoder
+"""
+import re
+import sys
+import struct
+
+from simplejson.scanner import make_scanner
+try:
+    from simplejson._speedups import scanstring as c_scanstring
+except ImportError:
+    c_scanstring = None
+
+__all__ = ['JSONDecoder']
+
+FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
+
+def _floatconstants():
+    _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
+    if sys.byteorder != 'big':
+        _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
+    nan, inf = struct.unpack('dd', _BYTES)
+    return nan, inf, -inf
+
+NaN, PosInf, NegInf = _floatconstants()
+
+
+def linecol(doc, pos):
+    lineno = doc.count('\n', 0, pos) + 1
+    if lineno == 1:
+        colno = pos
+    else:
+        colno = pos - doc.rindex('\n', 0, pos)
+    return lineno, colno
+
+
+def errmsg(msg, doc, pos, end=None):
+    # Note that this function is called from _speedups
+    lineno, colno = linecol(doc, pos)
+    if end is None:
+        #fmt = '{0}: line {1} column {2} (char {3})'
+        #return fmt.format(msg, lineno, colno, pos)
+        fmt = '%s: line %d column %d (char %d)'
+        return fmt % (msg, lineno, colno, pos)
+    endlineno, endcolno = linecol(doc, end)
+    #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
+    #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
+    fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
+    return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
+
+
+_CONSTANTS = {
+    '-Infinity': NegInf,
+    'Infinity': PosInf,
+    'NaN': NaN,
+}
+
+STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
+BACKSLASH = {
+    '"': u'"', '\\': u'\\', '/': u'/',
+    'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
+}
+
+DEFAULT_ENCODING = "utf-8"
+
+def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
+    """Scan the string s for a JSON string. End is the index of the
+    character in s after the quote that started the JSON string.
+    Unescapes all valid JSON string escape sequences and raises ValueError
+    on attempt to decode an invalid string. If strict is False then literal
+    control characters are allowed in the string.
+    
+    Returns a tuple of the decoded string and the index of the character in s
+    after the end quote."""
+    if encoding is None:
+        encoding = DEFAULT_ENCODING
+    chunks = []
+    _append = chunks.append
+    begin = end - 1
+    while 1:
+        chunk = _m(s, end)
+        if chunk is None:
+            raise ValueError(
+                errmsg("Unterminated string starting at", s, begin))
+        end = chunk.end()
+        content, terminator = chunk.groups()
+        # Content is contains zero or more unescaped string characters
+        if content:
+            if not isinstance(content, unicode):
+                content = unicode(content, encoding)
+            _append(content)
+        # Terminator is the end of string, a literal control character,
+        # or a backslash denoting that an escape sequence follows
+        if terminator == '"':
+            break
+        elif terminator != '\\':
+            if strict:
+                msg = "Invalid control character %r at" % (terminator,)
+                #msg = "Invalid control character {0!r} at".format(terminator)
+                raise ValueError(errmsg(msg, s, end))
+            else:
+                _append(terminator)
+                continue
+        try:
+            esc = s[end]
+        except IndexError:
+            raise ValueError(
+                errmsg("Unterminated string starting at", s, begin))
+        # If not a unicode escape sequence, must be in the lookup table
+        if esc != 'u':
+            try:
+                char = _b[esc]
+            except KeyError:
+                msg = "Invalid \\escape: " + repr(esc)
+                raise ValueError(errmsg(msg, s, end))
+            end += 1
+        else:
+            # Unicode escape sequence
+            esc = s[end + 1:end + 5]
+            next_end = end + 5
+            if len(esc) != 4:
+                msg = "Invalid \\uXXXX escape"
+                raise ValueError(errmsg(msg, s, end))
+            uni = int(esc, 16)
+            # Check for surrogate pair on UCS-4 systems
+            if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
+                msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
+                if not s[end + 5:end + 7] == '\\u':
+                    raise ValueError(errmsg(msg, s, end))
+                esc2 = s[end + 7:end + 11]
+                if len(esc2) != 4:
+                    raise ValueError(errmsg(msg, s, end))
+                uni2 = int(esc2, 16)
+                uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
+                next_end += 6
+            char = unichr(uni)
+            end = next_end
+        # Append the unescaped character
+        _append(char)
+    return u''.join(chunks), end
+
+
+# Use speedup if available
+scanstring = c_scanstring or py_scanstring
+
+WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
+WHITESPACE_STR = ' \t\n\r'
+
+def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
+    pairs = {}
+    # Use a slice to prevent IndexError from being raised, the following
+    # check will raise a more specific ValueError if the string is empty
+    nextchar = s[end:end + 1]
+    # Normally we expect nextchar == '"'
+    if nextchar != '"':
+        if nextchar in _ws:
+            end = _w(s, end).end()
+            nextchar = s[end:end + 1]
+        # Trivial empty object
+        if nextchar == '}':
+            return pairs, end + 1
+        elif nextchar != '"':
+            raise ValueError(errmsg("Expecting property name", s, end))
+    end += 1
+    while True:
+        key, end = scanstring(s, end, encoding, strict)
+
+        # To skip some function call overhead we optimize the fast paths where
+        # the JSON key separator is ": " or just ":".
+        if s[end:end + 1] != ':':
+            end = _w(s, end).end()
+            if s[end:end + 1] != ':':
+                raise ValueError(errmsg("Expecting : delimiter", s, end))
+
+        end += 1
+
+        try:
+            if s[end] in _ws:
+                end += 1
+                if s[end] in _ws:
+                    end = _w(s, end + 1).end()
+        except IndexError:
+            pass
+
+        try:
+            value, end = scan_once(s, end)
+        except StopIteration:
+            raise ValueError(errmsg("Expecting object", s, end))
+        pairs[key] = value
+
+        try:
+            nextchar = s[end]
+            if nextchar in _ws:
+                end = _w(s, end + 1).end()
+                nextchar = s[end]
+        except IndexError:
+            nextchar = ''
+        end += 1
+
+        if nextchar == '}':
+            break
+        elif nextchar != ',':
+            raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
+
+        try:
+            nextchar = s[end]
+            if nextchar in _ws:
+                end += 1
+                nextchar = s[end]
+                if nextchar in _ws:
+                    end = _w(s, end + 1).end()
+                    nextchar = s[end]
+        except IndexError:
+            nextchar = ''
+
+        end += 1
+        if nextchar != '"':
+            raise ValueError(errmsg("Expecting property name", s, end - 1))
+
+    if object_hook is not None:
+        pairs = object_hook(pairs)
+    return pairs, end
+
+def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
+    values = []
+    nextchar = s[end:end + 1]
+    if nextchar in _ws:
+        end = _w(s, end + 1).end()
+        nextchar = s[end:end + 1]
+    # Look-ahead for trivial empty array
+    if nextchar == ']':
+        return values, end + 1
+    _append = values.append
+    while True:
+        try:
+            value, end = scan_once(s, end)
+        except StopIteration:
+            raise ValueError(errmsg("Expecting object", s, end))
+        _append(value)
+        nextchar = s[end:end + 1]
+        if nextchar in _ws:
+            end = _w(s, end + 1).end()
+            nextchar = s[end:end + 1]
+        end += 1
+        if nextchar == ']':
+            break
+        elif nextchar != ',':
+            raise ValueError(errmsg("Expecting , delimiter", s, end))
+
+        try:
+            if s[end] in _ws:
+                end += 1
+                if s[end] in _ws:
+                    end = _w(s, end + 1).end()
+        except IndexError:
+            pass
+
+    return values, end
+
+class JSONDecoder(object):
+    """Simple JSON <http://json.org> decoder
+
+    Performs the following translations in decoding by default:
+
+    +---------------+-------------------+
+    | JSON          | Python            |
+    +===============+===================+
+    | object        | dict              |
+    +---------------+-------------------+
+    | array         | list              |
+    +---------------+-------------------+
+    | string        | unicode           |
+    +---------------+-------------------+
+    | number (int)  | int, long         |
+    +---------------+-------------------+
+    | number (real) | float             |
+    +---------------+-------------------+
+    | true          | True              |
+    +---------------+-------------------+
+    | false         | False             |
+    +---------------+-------------------+
+    | null          | None              |
+    +---------------+-------------------+
+
+    It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
+    their corresponding ``float`` values, which is outside the JSON spec.
+
+    """
+
+    def __init__(self, encoding=None, object_hook=None, parse_float=None,
+            parse_int=None, parse_constant=None, strict=True):
+        """``encoding`` determines the encoding used to interpret any ``str``
+        objects decoded by this instance (utf-8 by default).  It has no
+        effect when decoding ``unicode`` objects.
+
+        Note that currently only encodings that are a superset of ASCII work,
+        strings of other encodings should be passed in as ``unicode``.
+
+        ``object_hook``, if specified, will be called with the result
+        of every JSON object decoded and its return value will be used in
+        place of the given ``dict``.  This can be used to provide custom
+        deserializations (e.g. to support JSON-RPC class hinting).
+
+        ``parse_float``, if specified, will be called with the string
+        of every JSON float to be decoded. By default this is equivalent to
+        float(num_str). This can be used to use another datatype or parser
+        for JSON floats (e.g. decimal.Decimal).
+
+        ``parse_int``, if specified, will be called with the string
+        of every JSON int to be decoded. By default this is equivalent to
+        int(num_str). This can be used to use another datatype or parser
+        for JSON integers (e.g. float).
+
+        ``parse_constant``, if specified, will be called with one of the
+        following strings: -Infinity, Infinity, NaN.
+        This can be used to raise an exception if invalid JSON numbers
+        are encountered.
+
+        """
+        self.encoding = encoding
+        self.object_hook = object_hook
+        self.parse_float = parse_float or float
+        self.parse_int = parse_int or int
+        self.parse_constant = parse_constant or _CONSTANTS.__getitem__
+        self.strict = strict
+        self.parse_object = JSONObject
+        self.parse_array = JSONArray
+        self.parse_string = scanstring
+        self.scan_once = make_scanner(self)
+
+    def decode(self, s, _w=WHITESPACE.match):
+        """Return the Python representation of ``s`` (a ``str`` or ``unicode``
+        instance containing a JSON document)
+
+        """
+        obj, end = self.raw_decode(s, idx=_w(s, 0).end())
+        end = _w(s, end).end()
+        if end != len(s):
+            raise ValueError(errmsg("Extra data", s, end, len(s)))
+        return obj
+
+    def raw_decode(self, s, idx=0):
+        """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
+        with a JSON document) and return a 2-tuple of the Python
+        representation and the index in ``s`` where the document ended.
+
+        This can be used to decode a JSON document from a string that may
+        have extraneous data at the end.
+
+        """
+        try:
+            obj, end = self.scan_once(s, idx)
+        except StopIteration:
+            raise ValueError("No JSON object could be decoded")
+        return obj, end

simplejson/encoder.py

@@ -0,0 +1,440 @@
+"""Implementation of JSONEncoder
+"""
+import re
+
+try:
+    from simplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii
+except ImportError:
+    c_encode_basestring_ascii = None
+try:
+    from simplejson._speedups import make_encoder as c_make_encoder
+except ImportError:
+    c_make_encoder = None
+
+ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
+ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
+HAS_UTF8 = re.compile(r'[\x80-\xff]')
+ESCAPE_DCT = {
+    '\\': '\\\\',
+    '"': '\\"',
+    '\b': '\\b',
+    '\f': '\\f',
+    '\n': '\\n',
+    '\r': '\\r',
+    '\t': '\\t',
+}
+for i in range(0x20):
+    #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+
+# Assume this produces an infinity on all machines (probably not guaranteed)
+INFINITY = float('1e66666')
+FLOAT_REPR = repr
+
+def encode_basestring(s):
+    """Return a JSON representation of a Python string
+
+    """
+    def replace(match):
+        return ESCAPE_DCT[match.group(0)]
+    return '"' + ESCAPE.sub(replace, s) + '"'
+
+
+def py_encode_basestring_ascii(s):
+    """Return an ASCII-only JSON representation of a Python string
+
+    """
+    if isinstance(s, str) and HAS_UTF8.search(s) is not None:
+        s = s.decode('utf-8')
+    def replace(match):
+        s = match.group(0)
+        try:
+            return ESCAPE_DCT[s]
+        except KeyError:
+            n = ord(s)
+            if n < 0x10000:
+                #return '\\u{0:04x}'.format(n)
+                return '\\u%04x' % (n,)
+            else:
+                # surrogate pair
+                n -= 0x10000
+                s1 = 0xd800 | ((n >> 10) & 0x3ff)
+                s2 = 0xdc00 | (n & 0x3ff)
+                #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
+
+
+encode_basestring_ascii = c_encode_basestring_ascii or py_encode_basestring_ascii
+
+class JSONEncoder(object):
+    """Extensible JSON <http://json.org> encoder for Python data structures.
+
+    Supports the following objects and types by default:
+
+    +-------------------+---------------+
+    | Python            | JSON          |
+    +===================+===============+
+    | dict              | object        |
+    +-------------------+---------------+
+    | list, tuple       | array         |
+    +-------------------+---------------+
+    | str, unicode      | string        |
+    +-------------------+---------------+
+    | int, long, float  | number        |
+    +-------------------+---------------+
+    | True              | true          |
+    +-------------------+---------------+
+    | False             | false         |
+    +-------------------+---------------+
+    | None              | null          |
+    +-------------------+---------------+
+
+    To extend this to recognize other objects, subclass and implement a
+    ``.default()`` method with another method that returns a serializable
+    object for ``o`` if possible, otherwise it should call the superclass
+    implementation (to raise ``TypeError``).
+
+    """
+    item_separator = ', '
+    key_separator = ': '
+    def __init__(self, skipkeys=False, ensure_ascii=True,
+            check_circular=True, allow_nan=True, sort_keys=False,
+            indent=None, separators=None, encoding='utf-8', default=None):
+        """Constructor for JSONEncoder, with sensible defaults.
+
+        If skipkeys is false, then it is a TypeError to attempt
+        encoding of keys that are not str, int, long, float or None.  If
+        skipkeys is True, such items are simply skipped.
+
+        If ensure_ascii is true, the output is guaranteed to be str
+        objects with all incoming unicode characters escaped.  If
+        ensure_ascii is false, the output will be unicode object.
+
+        If check_circular is true, then lists, dicts, and custom encoded
+        objects will be checked for circular references during encoding to
+        prevent an infinite recursion (which would cause an OverflowError).
+        Otherwise, no such check takes place.
+
+        If allow_nan is true, then NaN, Infinity, and -Infinity will be
+        encoded as such.  This behavior is not JSON specification compliant,
+        but is consistent with most JavaScript based encoders and decoders.
+        Otherwise, it will be a ValueError to encode such floats.
+
+        If sort_keys is true, then the output of dictionaries will be
+        sorted by key; this is useful for regression tests to ensure
+        that JSON serializations can be compared on a day-to-day basis.
+
+        If indent is a non-negative integer, then JSON array
+        elements and object members will be pretty-printed with that
+        indent level.  An indent level of 0 will only insert newlines.
+        None is the most compact representation.
+
+        If specified, separators should be a (item_separator, key_separator)
+        tuple.  The default is (', ', ': ').  To get the most compact JSON
+        representation you should specify (',', ':') to eliminate whitespace.
+
+        If specified, default is a function that gets called for objects
+        that can't otherwise be serialized.  It should return a JSON encodable
+        version of the object or raise a ``TypeError``.
+
+        If encoding is not None, then all input strings will be
+        transformed into unicode using that encoding prior to JSON-encoding.
+        The default is UTF-8.
+
+        """
+
+        self.skipkeys = skipkeys
+        self.ensure_ascii = ensure_ascii
+        self.check_circular = check_circular
+        self.allow_nan = allow_nan
+        self.sort_keys = sort_keys
+        self.indent = indent
+        if separators is not None:
+            self.item_separator, self.key_separator = separators
+        if default is not None:
+            self.default = default
+        self.encoding = encoding
+
+    def default(self, o):
+        """Implement this method in a subclass such that it returns
+        a serializable object for ``o``, or calls the base implementation
+        (to raise a ``TypeError``).
+
+        For example, to support arbitrary iterators, you could
+        implement default like this::
+
+            def default(self, o):
+                try:
+                    iterable = iter(o)
+                except TypeError:
+                    pass
+                else:
+                    return list(iterable)
+                return JSONEncoder.default(self, o)
+
+        """
+        raise TypeError(repr(o) + " is not JSON serializable")
+
+    def encode(self, o):
+        """Return a JSON string representation of a Python data structure.
+
+        >>> JSONEncoder().encode({"foo": ["bar", "baz"]})
+        '{"foo": ["bar", "baz"]}'
+
+        """
+        # This is for extremely simple cases and benchmarks.
+        if isinstance(o, basestring):
+            if isinstance(o, str):
+                _encoding = self.encoding
+                if (_encoding is not None
+                        and not (_encoding == 'utf-8')):
+                    o = o.decode(_encoding)
+            if self.ensure_ascii:
+                return encode_basestring_ascii(o)
+            else:
+                return encode_basestring(o)
+        # This doesn't pass the iterator directly to ''.join() because the
+        # exceptions aren't as detailed.  The list call should be roughly
+        # equivalent to the PySequence_Fast that ''.join() would do.
+        chunks = self.iterencode(o, _one_shot=True)
+        if not isinstance(chunks, (list, tuple)):
+            chunks = list(chunks)
+        return ''.join(chunks)
+
+    def iterencode(self, o, _one_shot=False):
+        """Encode the given object and yield each string
+        representation as available.
+
+        For example::
+
+            for chunk in JSONEncoder().iterencode(bigobject):
+                mysocket.write(chunk)
+
+        """
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            _encoder = encode_basestring_ascii
+        else:
+            _encoder = encode_basestring
+        if self.encoding != 'utf-8':
+            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
+                if isinstance(o, str):
+                    o = o.decode(_encoding)
+                return _orig_encoder(o)
+
+        def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
+            # Check for specials.  Note that this type of test is processor- and/or
+            # platform-specific, so do tests which don't depend on the internals.
+
+            if o != o:
+                text = 'NaN'
+            elif o == _inf:
+                text = 'Infinity'
+            elif o == _neginf:
+                text = '-Infinity'
+            else:
+                return _repr(o)
+
+            if not allow_nan:
+                raise ValueError(
+                    "Out of range float values are not JSON compliant: " +
+                    repr(o))
+
+            return text
+
+
+        if _one_shot and c_make_encoder is not None and not self.indent and not self.sort_keys:
+            _iterencode = c_make_encoder(
+                markers, self.default, _encoder, self.indent,
+                self.key_separator, self.item_separator, self.sort_keys,
+                self.skipkeys, self.allow_nan)
+        else:
+            _iterencode = _make_iterencode(
+                markers, self.default, _encoder, self.indent, floatstr,
+                self.key_separator, self.item_separator, self.sort_keys,
+                self.skipkeys, _one_shot)
+        return _iterencode(o, 0)
+
+def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
+        ## HACK: hand-optimized bytecode; turn globals into locals
+        False=False,
+        True=True,
+        ValueError=ValueError,
+        basestring=basestring,
+        dict=dict,
+        float=float,
+        id=id,
+        int=int,
+        isinstance=isinstance,
+        list=list,
+        long=long,
+        str=str,
+        tuple=tuple,
+    ):
+
+    def _iterencode_list(lst, _current_indent_level):
+        if not lst:
+            yield '[]'
+            return
+        if markers is not None:
+            markerid = id(lst)
+            if markerid in markers:
+                raise ValueError("Circular reference detected")
+            markers[markerid] = lst
+        buf = '['
+        if _indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
+            separator = _item_separator + newline_indent
+            buf += newline_indent
+        else:
+            newline_indent = None
+            separator = _item_separator
+        first = True
+        for value in lst:
+            if first:
+                first = False
+            else:
+                buf = separator
+            if isinstance(value, basestring):
+                yield buf + _encoder(value)
+            elif value is None:
+                yield buf + 'null'
+            elif value is True:
+                yield buf + 'true'
+            elif value is False:
+                yield buf + 'false'
+            elif isinstance(value, (int, long)):
+                yield buf + str(value)
+            elif isinstance(value, float):
+                yield buf + _floatstr(value)
+            else:
+                yield buf
+                if isinstance(value, (list, tuple)):
+                    chunks = _iterencode_list(value, _current_indent_level)
+                elif isinstance(value, dict):
+                    chunks = _iterencode_dict(value, _current_indent_level)
+                else:
+                    chunks = _iterencode(value, _current_indent_level)
+                for chunk in chunks:
+                    yield chunk
+        if newline_indent is not None:
+            _current_indent_level -= 1
+            yield '\n' + (' ' * (_indent * _current_indent_level))
+        yield ']'
+        if markers is not None:
+            del markers[markerid]
+
+    def _iterencode_dict(dct, _current_indent_level):
+        if not dct:
+            yield '{}'
+            return
+        if markers is not None:
+            markerid = id(dct)
+            if markerid in markers:
+                raise ValueError("Circular reference detected")
+            markers[markerid] = dct
+        yield '{'
+        if _indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
+            item_separator = _item_separator + newline_indent
+            yield newline_indent
+        else:
+            newline_indent = None
+            item_separator = _item_separator
+        first = True
+        if _sort_keys:
+            items = dct.items()
+            items.sort(key=lambda kv: kv[0])
+        else:
+            items = dct.iteritems()
+        for key, value in items:
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = _floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif _skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            if first:
+                first = False
+            else:
+                yield item_separator
+            yield _encoder(key)
+            yield _key_separator
+            if isinstance(value, basestring):
+                yield _encoder(value)
+            elif value is None:
+                yield 'null'
+            elif value is True:
+                yield 'true'
+            elif value is False:
+                yield 'false'
+            elif isinstance(value, (int, long)):
+                yield str(value)
+            elif isinstance(value, float):
+                yield _floatstr(value)
+            else:
+                if isinstance(value, (list, tuple)):
+                    chunks = _iterencode_list(value, _current_indent_level)
+                elif isinstance(value, dict):
+                    chunks = _iterencode_dict(value, _current_indent_level)
+                else:
+                    chunks = _iterencode(value, _current_indent_level)
+                for chunk in chunks:
+                    yield chunk
+        if newline_indent is not None:
+            _current_indent_level -= 1
+            yield '\n' + (' ' * (_indent * _current_indent_level))
+        yield '}'
+        if markers is not None:
+            del markers[markerid]
+
+    def _iterencode(o, _current_indent_level):
+        if isinstance(o, basestring):
+            yield _encoder(o)
+        elif o is None:
+            yield 'null'
+        elif o is True:
+            yield 'true'
+        elif o is False:
+            yield 'false'
+        elif isinstance(o, (int, long)):
+            yield str(o)
+        elif isinstance(o, float):
+            yield _floatstr(o)
+        elif isinstance(o, (list, tuple)):
+            for chunk in _iterencode_list(o, _current_indent_level):
+                yield chunk
+        elif isinstance(o, dict):
+            for chunk in _iterencode_dict(o, _current_indent_level):
+                yield chunk
+        else:
+            if markers is not None:
+                markerid = id(o)
+                if markerid in markers:
+                    raise ValueError("Circular reference detected")
+                markers[markerid] = o
+            o = _default(o)
+            for chunk in _iterencode(o, _current_indent_level):
+                yield chunk
+            if markers is not None:
+                del markers[markerid]
+
+    return _iterencode

simplejson/scanner.py

@@ -0,0 +1,65 @@
+"""JSON token scanner
+"""
+import re
+try:
+    from simplejson._speedups import make_scanner as c_make_scanner
+except ImportError:
+    c_make_scanner = None
+
+__all__ = ['make_scanner']
+
+NUMBER_RE = re.compile(
+    r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
+    (re.VERBOSE | re.MULTILINE | re.DOTALL))
+
+def py_make_scanner(context):
+    parse_object = context.parse_object
+    parse_array = context.parse_array
+    parse_string = context.parse_string
+    match_number = NUMBER_RE.match
+    encoding = context.encoding
+    strict = context.strict
+    parse_float = context.parse_float
+    parse_int = context.parse_int
+    parse_constant = context.parse_constant
+    object_hook = context.object_hook
+
+    def _scan_once(string, idx):
+        try:
+            nextchar = string[idx]
+        except IndexError:
+            raise StopIteration
+
+        if nextchar == '"':
+            return parse_string(string, idx + 1, encoding, strict)
+        elif nextchar == '{':
+            return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook)
+        elif nextchar == '[':
+            return parse_array((string, idx + 1), _scan_once)
+        elif nextchar == 'n' and string[idx:idx + 4] == 'null':
+            return None, idx + 4
+        elif nextchar == 't' and string[idx:idx + 4] == 'true':
+            return True, idx + 4
+        elif nextchar == 'f' and string[idx:idx + 5] == 'false':
+            return False, idx + 5
+
+        m = match_number(string, idx)
+        if m is not None:
+            integer, frac, exp = m.groups()
+            if frac or exp:
+                res = parse_float(integer + (frac or '') + (exp or ''))
+            else:
+                res = parse_int(integer)
+            return res, m.end()
+        elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
+            return parse_constant('NaN'), idx + 3
+        elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
+            return parse_constant('Infinity'), idx + 8
+        elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
+            return parse_constant('-Infinity'), idx + 9
+        else:
+            raise StopIteration
+
+    return _scan_once
+
+make_scanner = c_make_scanner or py_make_scanner

simplejson/tests/__init__.py

@@ -0,0 +1,23 @@
+import unittest
+import doctest
+
+def additional_tests():
+    import simplejson
+    import simplejson.encoder
+    import simplejson.decoder
+    suite = unittest.TestSuite()
+    for mod in (simplejson, simplejson.encoder, simplejson.decoder):
+        suite.addTest(doctest.DocTestSuite(mod))
+    suite.addTest(doctest.DocFileSuite('../../index.rst'))
+    return suite
+
+def main():
+    suite = additional_tests()
+    runner = unittest.TextTestRunner()
+    runner.run(suite)
+
+if __name__ == '__main__':
+    import os
+    import sys
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+    main()

simplejson/tests/test_check_circular.py

@@ -0,0 +1,30 @@
+from unittest import TestCase
+import simplejson as json
+
+def default_iterable(obj):
+    return list(obj)
+
+class TestCheckCircular(TestCase):
+    def test_circular_dict(self):
+        dct = {}
+        dct['a'] = dct
+        self.assertRaises(ValueError, json.dumps, dct)
+
+    def test_circular_list(self):
+        lst = []
+        lst.append(lst)
+        self.assertRaises(ValueError, json.dumps, lst)
+
+    def test_circular_composite(self):
+        dct2 = {}
+        dct2['a'] = []
+        dct2['a'].append(dct2)
+        self.assertRaises(ValueError, json.dumps, dct2)
+
+    def test_circular_default(self):
+        json.dumps([set()], default=default_iterable)
+        self.assertRaises(TypeError, json.dumps, [set()])
+
+    def test_circular_off_default(self):
+        json.dumps([set()], default=default_iterable, check_circular=False)
+        self.assertRaises(TypeError, json.dumps, [set()], check_circular=False)

simplejson/tests/test_decode.py

@@ -0,0 +1,22 @@
+import decimal
+from unittest import TestCase
+
+import simplejson as json
+
+class TestDecode(TestCase):
+    def test_decimal(self):
+        rval = json.loads('1.1', parse_float=decimal.Decimal)
+        self.assert_(isinstance(rval, decimal.Decimal))
+        self.assertEquals(rval, decimal.Decimal('1.1'))
+
+    def test_float(self):
+        rval = json.loads('1', parse_int=float)
+        self.assert_(isinstance(rval, float))
+        self.assertEquals(rval, 1.0)
+
+    def test_decoder_optimizations(self):
+        # Several optimizations were made that skip over calls to
+        # the whitespace regex, so this test is designed to try and
+        # exercise the uncommon cases. The array cases are already covered.
+        rval = json.loads('{   "key"    :    "value"    ,  "k":"v"    }')
+        self.assertEquals(rval, {"key":"value", "k":"v"})

simplejson/tests/test_default.py

@@ -0,0 +1,9 @@
+from unittest import TestCase
+
+import simplejson as json
+
+class TestDefault(TestCase):
+    def test_default(self):
+        self.assertEquals(
+            json.dumps(type, default=repr),
+            json.dumps(repr(type)))

simplejson/tests/test_dump.py

@@ -0,0 +1,21 @@
+from unittest import TestCase
+from cStringIO import StringIO
+
+import simplejson as json
+
+class TestDump(TestCase):
+    def test_dump(self):
+        sio = StringIO()
+        json.dump({}, sio)
+        self.assertEquals(sio.getvalue(), '{}')
+
+    def test_dumps(self):
+        self.assertEquals(json.dumps({}), '{}')
+
+    def test_encode_truefalse(self):
+        self.assertEquals(json.dumps(
+                 {True: False, False: True}, sort_keys=True),
+                 '{"false": true, "true": false}')
+        self.assertEquals(json.dumps(
+                {2: 3.0, 4.0: 5L, False: 1, 6L: True, "7": 0}, sort_keys=True),
+                '{"false": 1, "2": 3.0, "4.0": 5, "6": true, "7": 0}')

simplejson/tests/test_encode_basestring_ascii.py

@@ -0,0 +1,38 @@
+from unittest import TestCase
+
+import simplejson.encoder
+
+CASES = [
+    (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'),
+    (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'),
+    (u'controls', '"controls"'),
+    (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'),
+    (u'{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'),
+    (u' s p a c e d ', '" s p a c e d "'),
+    (u'\U0001d120', '"\\ud834\\udd20"'),
+    (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'),
+    ('\xce\xb1\xce\xa9', '"\\u03b1\\u03a9"'),
+    (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'),
+    ('\xce\xb1\xce\xa9', '"\\u03b1\\u03a9"'),
+    (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'),
+    (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'),
+    (u"`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'),
+    (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'),
+    (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'),
+]
+
+class TestEncodeBaseStringAscii(TestCase):
+    def test_py_encode_basestring_ascii(self):
+        self._test_encode_basestring_ascii(simplejson.encoder.py_encode_basestring_ascii)
+
+    def test_c_encode_basestring_ascii(self):
+        if not simplejson.encoder.c_encode_basestring_ascii:
+            return
+        self._test_encode_basestring_ascii(simplejson.encoder.c_encode_basestring_ascii)
+
+    def _test_encode_basestring_ascii(self, encode_basestring_ascii):
+        fname = encode_basestring_ascii.__name__
+        for input_string, expect in CASES:
+            result = encode_basestring_ascii(input_string)
+            self.assertEquals(result, expect,
+                '%r != %r for %s(%r)' % (result, expect, fname, input_string))

simplejson/tests/test_fail.py

@@ -0,0 +1,76 @@
+from unittest import TestCase
+
+import simplejson as json
+
+# Fri Dec 30 18:57:26 2005
+JSONDOCS = [
+    # http://json.org/JSON_checker/test/fail1.json
+    '"A JSON payload should be an object or array, not a string."',
+    # http://json.org/JSON_checker/test/fail2.json
+    '["Unclosed array"',
+    # http://json.org/JSON_checker/test/fail3.json
+    '{unquoted_key: "keys must be quoted}',
+    # http://json.org/JSON_checker/test/fail4.json
+    '["extra comma",]',
+    # http://json.org/JSON_checker/test/fail5.json
+    '["double extra comma",,]',
+    # http://json.org/JSON_checker/test/fail6.json
+    '[   , "<-- missing value"]',
+    # http://json.org/JSON_checker/test/fail7.json
+    '["Comma after the close"],',
+    # http://json.org/JSON_checker/test/fail8.json
+    '["Extra close"]]',
+    # http://json.org/JSON_checker/test/fail9.json
+    '{"Extra comma": true,}',
+    # http://json.org/JSON_checker/test/fail10.json
+    '{"Extra value after close": true} "misplaced quoted value"',
+    # http://json.org/JSON_checker/test/fail11.json
+    '{"Illegal expression": 1 + 2}',
+    # http://json.org/JSON_checker/test/fail12.json
+    '{"Illegal invocation": alert()}',
+    # http://json.org/JSON_checker/test/fail13.json
+    '{"Numbers cannot have leading zeroes": 013}',
+    # http://json.org/JSON_checker/test/fail14.json
+    '{"Numbers cannot be hex": 0x14}',
+    # http://json.org/JSON_checker/test/fail15.json
+    '["Illegal backslash escape: \\x15"]',
+    # http://json.org/JSON_checker/test/fail16.json
+    '["Illegal backslash escape: \\\'"]',
+    # http://json.org/JSON_checker/test/fail17.json
+    '["Illegal backslash escape: \\017"]',
+    # http://json.org/JSON_checker/test/fail18.json
+    '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]',
+    # http://json.org/JSON_checker/test/fail19.json
+    '{"Missing colon" null}',
+    # http://json.org/JSON_checker/test/fail20.json
+    '{"Double colon":: null}',
+    # http://json.org/JSON_checker/test/fail21.json
+    '{"Comma instead of colon", null}',
+    # http://json.org/JSON_checker/test/fail22.json
+    '["Colon instead of comma": false]',
+    # http://json.org/JSON_checker/test/fail23.json
+    '["Bad value", truth]',
+    # http://json.org/JSON_checker/test/fail24.json
+    "['single quote']",
+    # http://code.google.com/p/simplejson/issues/detail?id=3
+    u'["A\u001FZ control characters in string"]',
+]
+
+SKIPS = {
+    1: "why not have a string payload?",
+    18: "spec doesn't specify any nesting limitations",
+}
+
+class TestFail(TestCase):
+    def test_failures(self):
+        for idx, doc in enumerate(JSONDOCS):
+            idx = idx + 1
+            if idx in SKIPS:
+                json.loads(doc)
+                continue
+            try:
+                json.loads(doc)
+            except ValueError:
+                pass
+            else:
+                self.fail("Expected failure for fail%d.json: %r" % (idx, doc))

simplejson/tests/test_float.py

@@ -0,0 +1,15 @@
+import math
+from unittest import TestCase
+
+import simplejson as json
+
+class TestFloat(TestCase):
+    def test_floats(self):
+        for num in [1617161771.7650001, math.pi, math.pi**100, math.pi**-100, 3.1]:
+            self.assertEquals(float(json.dumps(num)), num)
+            self.assertEquals(json.loads(json.dumps(num)), num)
+
+    def test_ints(self):
+        for num in [1, 1L, 1<<32, 1<<64]:
+            self.assertEquals(json.dumps(num), str(num))
+            self.assertEquals(int(json.dumps(num)), num)

simplejson/tests/test_indent.py

@@ -0,0 +1,41 @@
+from unittest import TestCase
+
+import simplejson as json
+import textwrap
+
+class TestIndent(TestCase):
+    def test_indent(self):
+        h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth',
+             {'nifty': 87}, {'field': 'yes', 'morefield': False} ]
+
+        expect = textwrap.dedent("""\
+        [
+          [
+            "blorpie"
+          ],
+          [
+            "whoops"
+          ],
+          [],
+          "d-shtaeou",
+          "d-nthiouh",
+          "i-vhbjkhnth",
+          {
+            "nifty": 87
+          },
+          {
+            "field": "yes",
+            "morefield": false
+          }
+        ]""")
+
+
+        d1 = json.dumps(h)
+        d2 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': '))
+
+        h1 = json.loads(d1)
+        h2 = json.loads(d2)
+
+        self.assertEquals(h1, h)
+        self.assertEquals(h2, h)
+        self.assertEquals(d2, expect)

simplejson/tests/test_pass1.py

@@ -0,0 +1,76 @@
+from unittest import TestCase
+
+import simplejson as json
+
+# from http://json.org/JSON_checker/test/pass1.json
+JSON = r'''
+[
+    "JSON Test Pattern pass1",
+    {"object with 1 member":["array with 1 element"]},
+    {},
+    [],
+    -42,
+    true,
+    false,
+    null,
+    {
+        "integer": 1234567890,
+        "real": -9876.543210,
+        "e": 0.123456789e-12,
+        "E": 1.234567890E+34,
+        "":  23456789012E666,
+        "zero": 0,
+        "one": 1,
+        "space": " ",
+        "quote": "\"",
+        "backslash": "\\",
+        "controls": "\b\f\n\r\t",
+        "slash": "/ & \/",
+        "alpha": "abcdefghijklmnopqrstuvwyz",
+        "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ",
+        "digit": "0123456789",
+        "special": "`1~!@#$%^&*()_+-={':[,]}|;.</>?",
+        "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A",
+        "true": true,
+        "false": false,
+        "null": null,
+        "array":[  ],
+        "object":{  },
+        "address": "50 St. James Street",
+        "url": "http://www.JSON.org/",
+        "comment": "// /* <!-- --",
+        "# -- --> */": " ",
+        " s p a c e d " :[1,2 , 3
+
+,
+
+4 , 5        ,          6           ,7        ],
+        "compact": [1,2,3,4,5,6,7],
+        "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}",
+        "quotes": "&#34; \u0022 %22 0x22 034 &#x22;",
+        "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?"
+: "A key can be any string"
+    },
+    0.5 ,98.6
+,
+99.44
+,
+
+1066
+
+
+,"rosebud"]
+'''
+
+class TestPass1(TestCase):
+    def test_parse(self):
+        # test in/out equivalence and parsing
+        res = json.loads(JSON)
+        out = json.dumps(res)
+        self.assertEquals(res, json.loads(out))
+        try:
+            json.dumps(res, allow_nan=False)
+        except ValueError:
+            pass
+        else:
+            self.fail("23456789012E666 should be out of range")

simplejson/tests/test_pass2.py

@@ -0,0 +1,14 @@
+from unittest import TestCase
+import simplejson as json
+
+# from http://json.org/JSON_checker/test/pass2.json
+JSON = r'''
+[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]]
+'''
+
+class TestPass2(TestCase):
+    def test_parse(self):
+        # test in/out equivalence and parsing
+        res = json.loads(JSON)
+        out = json.dumps(res)
+        self.assertEquals(res, json.loads(out))

simplejson/tests/test_pass3.py

@@ -0,0 +1,20 @@
+from unittest import TestCase
+
+import simplejson as json
+
+# from http://json.org/JSON_checker/test/pass3.json
+JSON = r'''
+{
+    "JSON Test Pattern pass3": {
+        "The outermost value": "must be an object or array.",
+        "In this test": "It is an object."
+    }
+}
+'''
+
+class TestPass3(TestCase):
+    def test_parse(self):
+        # test in/out equivalence and parsing
+        res = json.loads(JSON)
+        out = json.dumps(res)
+        self.assertEquals(res, json.loads(out))

simplejson/tests/test_recursion.py

@@ -0,0 +1,67 @@
+from unittest import TestCase
+
+import simplejson as json
+
+class JSONTestObject:
+    pass
+
+
+class RecursiveJSONEncoder(json.JSONEncoder):
+    recurse = False
+    def default(self, o):
+        if o is JSONTestObject:
+            if self.recurse:
+                return [JSONTestObject]
+            else:
+                return 'JSONTestObject'
+        return json.JSONEncoder.default(o)
+
+
+class TestRecursion(TestCase):
+    def test_listrecursion(self):
+        x = []
+        x.append(x)
+        try:
+            json.dumps(x)
+        except ValueError:
+            pass
+        else:
+            self.fail("didn't raise ValueError on list recursion")
+        x = []
+        y = [x]
+        x.append(y)
+        try:
+            json.dumps(x)
+        except ValueError:
+            pass
+        else:
+            self.fail("didn't raise ValueError on alternating list recursion")
+        y = []
+        x = [y, y]
+        # ensure that the marker is cleared
+        json.dumps(x)
+
+    def test_dictrecursion(self):
+        x = {}
+        x["test"] = x
+        try:
+            json.dumps(x)
+        except ValueError:
+            pass
+        else:
+            self.fail("didn't raise ValueError on dict recursion")
+        x = {}
+        y = {"a": x, "b": x}
+        # ensure that the marker is cleared
+        json.dumps(x)
+
+    def test_defaultrecursion(self):
+        enc = RecursiveJSONEncoder()
+        self.assertEquals(enc.encode(JSONTestObject), '"JSONTestObject"')
+        enc.recurse = True
+        try:
+            enc.encode(JSONTestObject)
+        except ValueError:
+            pass
+        else:
+            self.fail("didn't raise ValueError on default recursion")

simplejson/tests/test_scanstring.py

@@ -0,0 +1,111 @@
+import sys
+import decimal
+from unittest import TestCase
+
+import simplejson as json
+import simplejson.decoder
+
+class TestScanString(TestCase):
+    def test_py_scanstring(self):
+        self._test_scanstring(simplejson.decoder.py_scanstring)
+
+    def test_c_scanstring(self):
+        if not simplejson.decoder.c_scanstring:
+            return
+        self._test_scanstring(simplejson.decoder.c_scanstring)
+
+    def _test_scanstring(self, scanstring):
+        self.assertEquals(
+            scanstring('"z\\ud834\\udd20x"', 1, None, True),
+            (u'z\U0001d120x', 16))
+
+        if sys.maxunicode == 65535:
+            self.assertEquals(
+                scanstring(u'"z\U0001d120x"', 1, None, True),
+                (u'z\U0001d120x', 6))
+        else:
+            self.assertEquals(
+                scanstring(u'"z\U0001d120x"', 1, None, True),
+                (u'z\U0001d120x', 5))
+
+        self.assertEquals(
+            scanstring('"\\u007b"', 1, None, True),
+            (u'{', 8))
+
+        self.assertEquals(
+            scanstring('"A JSON payload should be an object or array, not a string."', 1, None, True),
+            (u'A JSON payload should be an object or array, not a string.', 60))
+
+        self.assertEquals(
+            scanstring('["Unclosed array"', 2, None, True),
+            (u'Unclosed array', 17))
+
+        self.assertEquals(
+            scanstring('["extra comma",]', 2, None, True),
+            (u'extra comma', 14))
+
+        self.assertEquals(
+            scanstring('["double extra comma",,]', 2, None, True),
+            (u'double extra comma', 21))
+
+        self.assertEquals(
+            scanstring('["Comma after the close"],', 2, None, True),
+            (u'Comma after the close', 24))
+
+        self.assertEquals(
+            scanstring('["Extra close"]]', 2, None, True),
+            (u'Extra close', 14))
+
+        self.assertEquals(
+            scanstring('{"Extra comma": true,}', 2, None, True),
+            (u'Extra comma', 14))
+
+        self.assertEquals(
+            scanstring('{"Extra value after close": true} "misplaced quoted value"', 2, None, True),
+            (u'Extra value after close', 26))
+
+        self.assertEquals(
+            scanstring('{"Illegal expression": 1 + 2}', 2, None, True),
+            (u'Illegal expression', 21))
+
+        self.assertEquals(
+            scanstring('{"Illegal invocation": alert()}', 2, None, True),
+            (u'Illegal invocation', 21))
+
+        self.assertEquals(
+            scanstring('{"Numbers cannot have leading zeroes": 013}', 2, None, True),
+            (u'Numbers cannot have leading zeroes', 37))
+
+        self.assertEquals(
+            scanstring('{"Numbers cannot be hex": 0x14}', 2, None, True),
+            (u'Numbers cannot be hex', 24))
+
+        self.assertEquals(
+            scanstring('[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', 21, None, True),
+            (u'Too deep', 30))
+
+        self.assertEquals(
+            scanstring('{"Missing colon" null}', 2, None, True),
+            (u'Missing colon', 16))
+
+        self.assertEquals(
+            scanstring('{"Double colon":: null}', 2, None, True),
+            (u'Double colon', 15))
+
+        self.assertEquals(
+            scanstring('{"Comma instead of colon", null}', 2, None, True),
+            (u'Comma instead of colon', 25))
+
+        self.assertEquals(
+            scanstring('["Colon instead of comma": false]', 2, None, True),
+            (u'Colon instead of comma', 25))
+
+        self.assertEquals(
+            scanstring('["Bad value", truth]', 2, None, True),
+            (u'Bad value', 12))
+
+    def test_issue3623(self):
+        self.assertRaises(ValueError, json.decoder.scanstring, "xxx", 1,
+                          "xxx")
+        self.assertRaises(UnicodeDecodeError,
+                          json.encoder.encode_basestring_ascii, "xx\xff")

simplejson/tests/test_separators.py

@@ -0,0 +1,42 @@
+import textwrap
+from unittest import TestCase
+
+import simplejson as json
+
+
+class TestSeparators(TestCase):
+    def test_separators(self):
+        h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth',
+             {'nifty': 87}, {'field': 'yes', 'morefield': False} ]
+
+        expect = textwrap.dedent("""\
+        [
+          [
+            "blorpie"
+          ] ,
+          [
+            "whoops"
+          ] ,
+          [] ,
+          "d-shtaeou" ,
+          "d-nthiouh" ,
+          "i-vhbjkhnth" ,
+          {
+            "nifty" : 87
+          } ,
+          {
+            "field" : "yes" ,
+            "morefield" : false
+          }
+        ]""")
+
+
+        d1 = json.dumps(h)
+        d2 = json.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : '))
+
+        h1 = json.loads(d1)
+        h2 = json.loads(d2)
+
+        self.assertEquals(h1, h)
+        self.assertEquals(h2, h)
+        self.assertEquals(d2, expect)

simplejson/tests/test_unicode.py

@@ -0,0 +1,64 @@
+from unittest import TestCase
+
+import simplejson as json
+
+class TestUnicode(TestCase):
+    def test_encoding1(self):
+        encoder = json.JSONEncoder(encoding='utf-8')
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        s = u.encode('utf-8')
+        ju = encoder.encode(u)
+        js = encoder.encode(s)
+        self.assertEquals(ju, js)
+
+    def test_encoding2(self):
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        s = u.encode('utf-8')
+        ju = json.dumps(u, encoding='utf-8')
+        js = json.dumps(s, encoding='utf-8')
+        self.assertEquals(ju, js)
+
+    def test_encoding3(self):
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        j = json.dumps(u)
+        self.assertEquals(j, '"\\u03b1\\u03a9"')
+
+    def test_encoding4(self):
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        j = json.dumps([u])
+        self.assertEquals(j, '["\\u03b1\\u03a9"]')
+
+    def test_encoding5(self):
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        j = json.dumps(u, ensure_ascii=False)
+        self.assertEquals(j, u'"%s"' % (u,))
+
+    def test_encoding6(self):
+        u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}'
+        j = json.dumps([u], ensure_ascii=False)
+        self.assertEquals(j, u'["%s"]' % (u,))
+
+    def test_big_unicode_encode(self):
+        u = u'\U0001d120'
+        self.assertEquals(json.dumps(u), '"\\ud834\\udd20"')
+        self.assertEquals(json.dumps(u, ensure_ascii=False), u'"\U0001d120"')
+
+    def test_big_unicode_decode(self):
+        u = u'z\U0001d120x'
+        self.assertEquals(json.loads('"' + u + '"'), u)
+        self.assertEquals(json.loads('"z\\ud834\\udd20x"'), u)
+
+    def test_unicode_decode(self):
+        for i in range(0, 0xd7ff):
+            u = unichr(i)
+            s = '"\\u%04x"' % (i,)
+            self.assertEquals(json.loads(s), u)
+
+    def test_default_encoding(self):
+        self.assertEquals(json.loads(u'{"a": "\xe9"}'.encode('utf-8')),
+            {'a': u'\xe9'})
+
+    def test_unicode_preservation(self):
+        self.assertEquals(type(json.loads(u'""')), unicode)
+        self.assertEquals(type(json.loads(u'"a"')), unicode)
+        self.assertEquals(type(json.loads(u'["a"]')[0]), unicode)

simplejson/tool.py

@@ -0,0 +1,37 @@
+r"""Command-line tool to validate and pretty-print JSON
+
+Usage::
+
+    $ echo '{"json":"obj"}' | python -m simplejson.tool
+    {
+        "json": "obj"
+    }
+    $ echo '{ 1.2:3.4}' | python -m simplejson.tool
+    Expecting property name: line 1 column 2 (char 2)
+
+"""
+import sys
+import simplejson
+
+def main():
+    if len(sys.argv) == 1:
+        infile = sys.stdin
+        outfile = sys.stdout
+    elif len(sys.argv) == 2:
+        infile = open(sys.argv[1], 'rb')
+        outfile = sys.stdout
+    elif len(sys.argv) == 3:
+        infile = open(sys.argv[1], 'rb')
+        outfile = open(sys.argv[2], 'wb')
+    else:
+        raise SystemExit(sys.argv[0] + " [infile [outfile]]")
+    try:
+        obj = simplejson.load(infile)
+    except ValueError, e:
+        raise SystemExit(e)
+    simplejson.dump(obj, outfile, sort_keys=True, indent=4)
+    outfile.write('\n')
+
+
+if __name__ == '__main__':
+    main()

utils/remover.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+remover.py
+
+Created by Roman on 2010-06-20.
+Copyright (c) 2010 __MyCompanyName__. All rights reserved.
+"""
+
+import datetime
+import logging
+
+from google.appengine.ext.webapp import util
+from google.appengine.ext import webapp
+from google.appengine.api import users
+
+from ffstorage import *
+
+class Remover(webapp.RequestHandler):
+	def get(self):
+		logging.debug("Starting r3m0v3r")
+		user = users.get_current_user()
+		logging.debug("Working as user %s" % user)
+		theDate = datetime.date.today() - datetime.timedelta(days=2)
+		logging.debug("Will delete stuff older than %s" % theDate)
+
+		fics = DownloadedFanfic.all()
+		fics.order("date")
+		
+		results = fics.fetch(50)
+		
+		
+		logging.debug([x.name for x in results])
+		                        
+		num = 0
+		for d in results:
+#			d.blob = None
+#			d.cleared = True
+			d.delete()
+			num = num + 1
+		logging.info('Deleted instances: %d' % num)
+		self.response.out.write('Deleted instances: %d' % num)
+		
+
+def main():
+	application = webapp.WSGIApplication([('/r3m0v3r', Remover)],
+                                               debug=False)
+	util.run_wsgi_app(application)
+
+
+if __name__ == '__main__':
+	logging.getLogger().setLevel(logging.DEBUG)
+	main()