Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
M
Mozrepl-Python
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
1
Issues
1
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Incidents
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Melroy van den Berg
Mozrepl-Python
Commits
a37c51ea
Commit
a37c51ea
authored
Jul 28, 2017
by
Melroy van den Berg
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Split Marktplaats part to seperate module, delivery method selection works!
parent
59150476
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
26 additions
and
71 deletions
+26
-71
main.py
main.py
+9
-10
mozrepl.py
mozrepl.py
+17
-61
No files found.
main.py
View file @
a37c51ea
#!/usr/bin/env python3
# Marktplaats batch script
# By: Melroy van den Berg
from
mozrepl
import
MozRepl
from
marktplaats
import
Marktplaats
# https://www.marktplaats.nl/syi/201/228/plaatsAdvertentie.html?bucket=38&complete=true&origin=HEADER&l1=201&l2=228&title=Test&attribute[condition]=31&attribute[level]=1950&attribute[orientation]=1955&price.bidding=free-bidding&description=blah&price.value=25,50&acceptPaypal=true&attribute[delivery]=35&shippingDetails.price=650&shippingDetails.provider=postnl&shippingDetails.productCode=3000&shippingDetails.catalogName=parcels&shippingDetails.averageWeight=5000
# Shipping trick only works in Chrome not in Firefox! Arrrg! Try setting hidden fields via javascript instead or other hacking when required.
# Shipping data: https://api.marktplaats.nl/v1/postnl/catalog
m
=
None
m
p
=
None
try
:
m
=
MozRepl
()
m
p
=
Marktplaats
()
# Is user loggedin?
login
=
m
.
check_marktplaats_login
()
login
=
m
p
.
check_marktplaats_login
()
if
login
==
0
:
# Not yet logged in
# TODO: target can maybe be used in future
m
.
load_page_blocking
(
"https://www.marktplaats.nl/account/login.html?target=https%3A%2F%2Fwww.marktplaats.nl%2F"
,
"https://www.marktplaats.nl/account/login.htm
l"
)
m
p
.
load_page_blocking
(
"https://www.marktplaats.nl/account/login.html?target=https%3A%2F%2Fwww.marktplaats.nl%2F"
,
"https://www.marktplaats.nl/account/login.html"
,
"https://www.marktplaats.nl/cookiewal
l"
)
print
(
"Succesful login (hopefully), continue..."
)
m
.
load_form_page_blocking
(
"https://www.marktplaats.nl/syi/201/228/plaatsAdvertentie.html?bucket=38&complete=true&origin=HEADER&l1=201&l2=228&title=Test&attribute[condition]=31&attribute[level]=1950&attribute[orientation]=1955&price.bidding=free-bidding&description=blah&price.value=25,50&acceptPaypal=true&attribute[delivery]=35&shippingDetails.price=650&shippingDetails.provider=postnl&shippingDetails.productCode=3000&shippingDetails.catalogName=parcels&shippingDetails.averageWeight=5000
"
)
m
p
.
load_form_page_blocking
(
"https://www.marktplaats.nl/syi/201/228/plaatsAdvertentie.html?bucket=38&complete=true&origin=HEADER&l1=201&l2=228&title=Test&attribute[condition]=31&attribute[level]=1950&attribute[orientation]=1955&price.bidding=free-bidding&description=blah&price.value=25,50&acceptPaypal=true&contactInformation.sellerName=Melroy&contactInformation.phoneNumber=06400&contactInformation.postCode=6605DW
"
)
except
(
KeyboardInterrupt
,
SystemExit
):
print
(
"Exit, bye bye."
)
if
m
:
if
m
p
:
# close connection
m
.
close
()
m
p
.
close
()
mozrepl.py
View file @
a37c51ea
...
...
@@ -13,7 +13,7 @@ import process
NR_RETRIES_DOCUMENT
=
1000
# 200 sec (3.3 min. time-out)/ 0.2 sec = 1000 tries
NR_RETRIES_FIREFOX
=
360
# 180 sec (3 min. time-out)/ 0.5 sec = 360 tries
class
MozRepl
:
class
MozRepl
(
object
)
:
def
__init__
(
self
):
"""
Contructor, only create a telnet connection
...
...
@@ -43,34 +43,7 @@ class MozRepl:
self
.
SOCKET_NAME
=
sockname
self
.
READ_UNTIL_CHARS
=
self
.
SOCKET_NAME
+
b
'>'
def
check_marktplaats_login
(
self
):
"""
Verifiy if user is already logged-in
"""
# Open marktplaats
self
.
open_url
(
"https://www.marktplaats.nl"
)
# Switch to the document active tab (web-page)
self
.
switch_to_content
()
# From this moment on, the write_data/read_data calls COULD be important to
# run in context of content (enter(content)), use read_data(true) when applicable and check the valid flag
# Check logout url availability
self
.
write_data
(
b
"function checkLogin(){for(var n=0,t=document.links.length;n<t;n++)if(document.links[n].href.startsWith(
\"
https://auth.marktplaats.nl/accounts/authentication/logout
\"
))return 1;return null}"
)
self
.
skip_data
()
# Execute the javascript function above
self
.
write_data
(
b
"checkLogin()"
)
# 1 is logout url found, 0 is not found
(
loggedin
,
valid
)
=
self
.
read_data
()
if
loggedin
==
b
'1'
:
print
(
"Already logged in at Marktplaats."
)
#self.create_popup("U bent reeds ingelogd in Marktplaats.")
return
1
else
:
print
(
"First login into Markplaats!"
)
return
0
def
load_page_blocking
(
self
,
url
,
blocking_url
=
""
):
def
load_page_blocking
(
self
,
url
,
blocking_url
=
""
,
exception_url
=
None
):
"""
Load a website, busy waiting until submitted
url - page to load
...
...
@@ -89,31 +62,7 @@ class MozRepl:
blocking_url
=
url
blocking_url
=
b
'"'
+
blocking_url
.
encode
(
'utf-8'
)
# Blocking wait until the current URL is different from the 'blocking url'
self
.
wait_until_url_changed
(
blocking_url
)
def
load_form_page_blocking
(
self
,
url
,
blocking_url
=
""
):
"""
Load the markplaats 'Plaats advertensie' page, busy waiting until submitted
url - page to load
blocking_url - url (beginning of url) that waits until changed differently (blocking until changed to another URL)
"""
# Open url in current tab/window
self
.
open_url
(
url
)
# Switch to the document active tab (web-page)
self
.
switch_to_content
()
# From this moment on, the write_data/read_data calls COULD be important to
# run in context of content (enter(content)), use read_data(true) when applicable and check the valid flag
# Select "Ophalen of Verzenden" in delivery method in the dropdown form
self
.
change_delivery_method_to_sendpickup
()
# When no blocking url is defined use input URL instead
if
blocking_url
==
""
:
blocking_url
=
url
blocking_url
=
b
'"'
+
blocking_url
.
encode
(
'utf-8'
)
# Blocking wait until the current URL is different from the 'blocking url' (form submitted)
self
.
wait_until_url_changed
(
blocking_url
)
self
.
wait_until_url_changed
(
blocking_url
,
exception_url
)
def
wait_until_page_complete
(
self
):
"""
...
...
@@ -135,9 +84,12 @@ class MozRepl:
print
(
"ERROR: Too many retries. Do you have internet connection at all? Exit."
)
# Time-out
exit
(
1
)
def
wait_until_url_changed
(
self
,
blocking_url
):
def
wait_until_url_changed
(
self
,
blocking_url
,
exception_url
):
"""
Busy wait until the until the current URL is different from the 'blocking url'
blocking_url - Blocking url until this is different
exception_url - Where it should not trigger on (eg. cookie pop-up URL), which is a false positive trigger
"""
print_wait
=
False
while
True
:
...
...
@@ -146,8 +98,12 @@ class MozRepl:
(
cur_url
,
valid
)
=
self
.
read_data
(
True
)
# Only look at cur_url when there is valid data (data running in the current content)
if
valid
:
# Ignore the cookie url
if
not
cur_url
.
startswith
(
b
'"https://www.marktplaats.nl/cookiewall'
):
if
exception_url
is
not
None
:
if
cur_url
.
startswith
(
b
'"'
+
exception_url
.
encode
(
'utf-8'
)):
# Ignore the exception_url (if applicable)
valid
=
False
if
valid
:
if
not
cur_url
.
startswith
(
blocking_url
):
# Page, url is changed, exit wait
break
...
...
@@ -200,7 +156,7 @@ class MozRepl:
try
:
output
=
self
.
t
.
read_until
(
self
.
READ_UNTIL_CHARS
)[:
-
i
].
strip
()
except
OSError
as
e
:
print
(
"Connection problem. Did you disable the plugin or closed Firefox?
??
Exit"
)
print
(
"Connection problem. Did you disable the plugin or closed Firefox? Exit"
)
exit
(
1
)
# During a context switch (page change) repl will go back to home,
...
...
@@ -215,11 +171,11 @@ class MozRepl:
valid
=
False
return
(
output
,
valid
)
def
change_delivery_method_to_sendpickup
(
self
):
def
set_hidden_input_field_form
(
self
,
form_id
,
name
,
value
):
"""
Change the select dropbox item in Marktplaats ("Ophalen of Verzenden"), to index 3 == Ophalen of Verzenden
Append hidden input field to a form (id)
"""
self
.
write_data
(
b
"
document.getElementById('deliveryMethod').childNodes[1].childNodes[1].selectedIndex=3
"
)
self
.
write_data
(
b
"
input = document.createElement('input');input.setAttribute('type', 'hidden');input.setAttribute('name', '"
+
name
.
encode
(
'utf-8'
)
+
b
"');input.setAttribute('value', '"
+
value
.
encode
(
'utf-8'
)
+
b
"');document.getElementById('"
+
form_id
.
encode
(
'utf-8'
)
+
b
"').appendChild(input);
"
)
self
.
skip_data
()
def
switch_to_content
(
self
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment