Commit 1a37258e authored by Melroy van den Berg's avatar Melroy van den Berg

It just works now!

parent 8b7a8e11
......@@ -3,12 +3,22 @@ from mozrepl import Mozrepl
# https://www.marktplaats.nl/syi/201/228/plaatsAdvertentie.html?bucket=38&complete=true&origin=HEADER&l1=201&l2=228&title=Test&attribute[condition]=31&attribute[level]=1950&attribute[orientation]=1955&price.bidding=free-bidding&description=blah&price.value=25,50&acceptPaypal=true&attribute[delivery]=35&shippingDetails.price=650&shippingDetails.provider=postnl&shippingDetails.productCode=3000&shippingDetails.catalogName=parcels&shippingDetails.averageWeight=5000
# Shipping trick only works in Chrome not in Firefox! Arrrg
# Shipping trick only works in Chrome not in Firefox! Arrrg! Try setting hidden fields via javascript instead or other hacking when required.
# Shipping data: https://api.marktplaats.nl/v1/postnl/catalog
try:
m = Mozrepl()
# Is user loggedin?
login = m.check_marktplaats_login()
if login == 0:
# Not yet logged in
# TODO: target can maybe be used in future
m.load_page_blocking("https://www.marktplaats.nl/account/login.html?target=https%3A%2F%2Fwww.marktplaats.nl%2F", "https://www.marktplaats.nl/account/login.html")
print "Succesful login (hopefully), continue..."
# https://api.marktplaats.nl/v1/postnl/catalog
m = Mozrepl()
m.load_page_blocking("https://www.marktplaats.nl/syi/201/228/plaatsAdvertentie.html?bucket=38&origin=HEADER&title=")
m.close()
m.load_page_blocking("https://www.marktplaats.nl/syi/201/228/plaatsAdvertentie.html?bucket=38&complete=true&origin=HEADER&l1=201&l2=228&title=Test&attribute[condition]=31&attribute[level]=1950&attribute[orientation]=1955&price.bidding=free-bidding&description=blah&price.value=25,50&acceptPaypal=true&attribute[delivery]=35&shippingDetails.price=650&shippingDetails.provider=postnl&shippingDetails.productCode=3000&shippingDetails.catalogName=parcels&shippingDetails.averageWeight=5000")
except (KeyboardInterrupt, SystemExit):
print "Exit, bye bye."
if m:
# close connection
m.close()
#!/usr/bin/env python
#https://addons.mozilla.org/en-US/firefox/addon/mozrepl/
from time import sleep
import telnetlib
import socket
NR_RETRIES=800 # 240 sec (4 min.)/ 0.3 sec
READ_UNTIL_CHARS="repl1>"
class Mozrepl:
def __init__(self):
# telnet localhost 4242
"""
Contructor, only create a telnet connection
"""
# Default socket name (repl)
self.SOCKET_NAME = "repl"
# $ telnet localhost 4242
try:
self.t = telnetlib.Telnet("localhost", 4242)
except socket.error:
print "ERROR: Could not connect to Firefox. Did you install Mozrepl plugin in Firefox?\nIf so, did you also start the service. In the menu: Extra->Mozrepl->Start."
print "ERROR: Could not connect to Firefox. Did you install Mozrepl plugin in Firefox? (Install: https://addons.mozilla.org/nl/firefox/addon/mozrepl/).\nIf so, did you also start the service. In the menu: Extra->Mozrepl->Start."
exit(1)
conn_log = self.readAllData()
split_log = conn_log.splitlines()
sockname = split_log[-1].split(">")[0].strip()
if self.SOCKET_NAME != sockname:
self.SOCKET_NAME = sockname
self.READ_UNTIL_CHARS=self.SOCKET_NAME+">"
def check_marktplaats_login(self):
"""
Verifiy if user is already logged-in
"""
# Open marktplaats
self.open_url("https://www.marktplaats.nl")
# Blocking wait
self.wait_until_page_complete()
# Switch to the document active tab (web-page)
self.switchToContent()
# From this moment on, the writeData/readData calls COULD be important to
# run in context of content (enter(content)), use readData(true) when applicable and check the valid flag
# Check logout url availability
self.writeData("function checkLogin(){for(var n=0,t=document.links.length;n<t;n++)if(document.links[n].href.startsWith(\"https://auth.marktplaats.nl/accounts/authentication/logout\"))return 1;return null}")
self.skipData()
self.writeData("checkLogin()")
(loggedin, valid) = self.readData()
if loggedin == "1":
print "Already logged-in."
self.writeData("alert('U bent reeds ingelogd in Marktplaats.')")
return 1
else:
print "First login into Markplaats!"
return 0
def load_page_blocking(self, url):
def load_page_blocking(self, url, blocking_url=""):
"""
Load a website, busy waiting until submitted
url - page to load
blocking_url - url (beginning of url) that waits until changed differently (blocking until changed to another URL)
"""
# Open url in current tab/window
self.writeData("content.location.href='"+url.strip()+"'")
self.skipData()
self.open_url(url)
tries=0
while True:
# Get the page load state
self.writeData("document.readyState")
state = self.readData()
sleep(0.1)
if state == "\"complete\"":
# When completed wait a bit longer, in order to set repl again.
sleep(0.5)
break
tries+=1
if tries == NR_RETRIES:
print "ERROR: Too many retries. Do you have internet connection at all? Exit." # Time-out
exit(1)
# Blocking wait
self.wait_until_page_complete()
# Set a temporally variable
self.writeData("repl1.enter(content)")
self.skipData()
self.writeData("document.temp=1")
# Switch to the document active tab (web-page)
self.switchToContent()
# From this moment on, the writeData/readData calls COULD be important to
# run in context of content (enter(content)), use readData(true) when applicable and check the valid flag
# Experimental. TODO
# repl.execute('content').document.hidden1 = repl.execute('content').document.createElement("input")
......@@ -50,36 +78,98 @@ class Mozrepl:
# repl.execute('content').document.getElementById("syi-form").appendChild(document.hidden1)
# Polling until 'temp' is becoming undefined again, meaning it's back to creation content (context unloading)
if blocking_url == "":
# When no blocking url is defined use input URL instead
blocking_url = url
while True:
self.writeData("window.location.href")
# Be-sure we run in content always (see valid flag)
(cur_url, valid) = self.readData(True)
# Only look at cur_url when there is valid data
if valid:
if not cur_url.startswith('"' + blocking_url):
# Page, url is changed (page submitted?)
break
else:
print "Required page change not yet happend. Waiting for user."
sleep(3)
def wait_until_page_complete(self):
"""
Busy wait until the page is completly loaded
"""
tries=0
while True:
temp = self.writeData("document.temp")
temp = self.readData()
if temp != "1":
# Form is submitted or page is closed, we continue!
# Get the page load state
self.writeData("document.readyState")
(state, valid) = self.readData()
sleep(0.2)
if state == "\"complete\"":
# When completed wait a bit longer, in order to set repl again + HTML is updated fully, strange right?
sleep(2)
break
else:
print "Form not yet submitted"
sleep(3)
tries+=1
if tries == NR_RETRIES:
print "ERROR: Too many retries. Do you have internet connection at all? Exit." # Time-out
exit(1)
def open_url(self, url):
"""
Trigger a change of URL in current tab
"""
self.writeData("content.location.href='"+url.strip()+"'")
self.skipData()
def writeData(self, str_buffer):
"""
Write data via write. New line at the end.
"""
"""
self.t.write(str_buffer + "\n")
def readData(self):
def readData(self, inContentImportant=False):
"""
Read the buffer via read until, strip 'reply>' from the message
REMINDER: Don't forget to check the valid flag, when isContent=True!
input(s): inContentImportant - Should the readData only work when the content is entered (repl.enter(content)?
output(s): output - the buffer from repl itself.
valid - if valid is invalid (false), execute the previous writeData (followed by a readData) again.
"""
i = len(READ_UNTIL_CHARS)
return self.t.read_until(READ_UNTIL_CHARS)[:-i].strip()
valid=True
i = len(self.READ_UNTIL_CHARS)
output = self.t.read_until(self.READ_UNTIL_CHARS)[:-i].strip()
# During a context switch (page change) repl will go back to home,
# should the read command be part of the Document context (eg. is enter(content) required in order to work)
if inContentImportant:
# If a context uploading is seen, switch back to the content again: enter(content).
if output.startswith("Host context unloading"):
self.switchToContent()
# Mark as invalid, since the data is not what you expect to be
# re-execute the previous writeData() agaom/
valid=False
return (output,valid)
def switchToContent(self):
self.writeData(self.SOCKET_NAME + ".enter(content)")
self.skipData()
def readAllData(self):
return self.t.read_until("> ")
def skipData(self):
"""
Also read data until, but ignore the output,
get the buffer clean
"""
self.t.read_until(READ_UNTIL_CHARS)
self.t.read_until(self.READ_UNTIL_CHARS)
def close(self):
"""
Close active connection
"""
#self.writeData(self.SOCKET_NAME + ".quit()")
self.t.close()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment