diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e07c3fd26c68550e0dc72d5c204f5bc603f6e72..e36e66e9715eb5591e1c67be4ebfefd1f9e65c83 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,8 +41,10 @@ deploy: - sudo apt-get install -y python-pip #install pip, this is needed to install docker-compose in the next step - sudo pip install docker-compose #Install docker-compose with pip - replace "5000:5000" "5003:5000" -- docker-compose.yml #replace in docker-compose some settings. Variables will be replaced on execute. - - replace "domains=" "$domains" -- docker-compose.yml #The domains I use for my url shorter - - replace "recaptcha_private=" "$recaptcha_private" -- docker-compose.yml #Recaptcha keys for protecting the create form from attacks - - replace "recaptcha_public=" "$recaptcha_public" -- docker-compose.yml + - replace "domains=" "domains=$domains" -- docker-compose.yml #The domains I use for my url shorter + - replace "recaptcha_private=" "recaptcha_private=$recaptcha_private" -- docker-compose.yml #Recaptcha keys for protecting the create form from attacks + - replace "recaptcha_public=" "recaptcha_public=$recaptcha_public" -- docker-compose.yml + - replace "GITHUB_CLIENT_ID=" "GITHUB_CLIENT_ID=$GITHUB_CLIENT_ID" -- docker-compose.yml + - replace "GITHUB_CLIENT_SECRET=" "GITHUB_CLIENT_SECRET=$GITHUB_CLIENT_SECRET" -- docker-compose.yml - sudo docker-compose up -d #Start the new container environment: master \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3f7e3b5864b5e2911ac78edc0e57b3c4eb075006..e70e80ab39a1eb34e9465b3947f0126e8610b1c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,7 @@ COPY import.py /app/import.py COPY export.py /app/export.py COPY main.py /app/main.py COPY VERSION /app/VERISON +COPY requirements.txt /app/requirements.txt #Make a complete system update. apt-utils is needed for configuring packages, so we need to install it RUN apt update @@ -18,9 +19,7 @@ RUN apt install apt-utils -y RUN apt upgrade -y RUN apt clean -#Install pipreqs. This tool is used to make the requirements.txt file automatic. Afterwards install them. -RUN pip install pipreqs -RUN pipreqs . --force +#Install libraries RUN python3 -m pip install -r requirements.txt #Compile the Python files (for more speed) and replace the original files diff --git a/VERSION b/VERSION index e1df5de7ae8c0e5f2eb07df65f15a95766f6c9b0..a73b432544448dccf8814bfa830a722e78ba83c0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.4 \ No newline at end of file +1.5.2 \ No newline at end of file diff --git a/docker-compose-build.yml b/docker-compose-build.yml index fea1ea6aa14e20093e01c9601787a33039c9f51a..1b68a2239e41a806f2482cde3a3b1e8745a845b4 100644 --- a/docker-compose-build.yml +++ b/docker-compose-build.yml @@ -16,6 +16,7 @@ services: - recaptcha_private= #Please enter here your private Key for google recaptcha - recaptcha_public= #Please enter here your public Key for google recaptcha - host=0.0.0.0 #With this variable you can set the access ip range. 127.0.0.1 means you can only access it from the local network and 0.0.0.0 means everyone can access it. + - GITHUB_CLIENT_ID= #You have to set these two variables, if not the shorter will not run. To get the keys visit https://github.com/settings/developers and register a new oauth application. The callback path is /user/github-callback + - GITHUB_CLIENT_SECRET= volumes: - url_shorter_db: - \ No newline at end of file + url_shorter_db: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 3f6fa49b17a0c17bebb3125b2c47480a3fbdd6e3..012b261b4e4bac000759d7f1cb69a93d1c8a32a7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,8 @@ services: - recaptcha_private= #Please enter here your private Key for google recaptcha - recaptcha_public= #Please enter here your public Key for google recaptcha - host=0.0.0.0 #With this variable you can set the access ip range. 127.0.0.1 means you can only access it from the local network and 0.0.0.0 means everyone can access it. + - GITHUB_CLIENT_ID= #You have to set these two variables, if not the shorter will not run. To get the keys visit https://github.com/settings/developers and register a new oauth application. The callback path is /user/github-callback + - GITHUB_CLIENT_SECRET= volumes: url_shorter_db: \ No newline at end of file diff --git a/export.py b/export.py index 400463a6894a44c70686bf46f489d89b22fa9d5a..0d630fcff86fb823e22e48aa4ccece71ece25186 100644 --- a/export.py +++ b/export.py @@ -6,8 +6,9 @@ import sqlite3 create_table = """ CREATE TABLE WEB_URL( - LONG_URL TEXT NOT NULL - SHORT_URL TEXT NOT NULL + LONG_URL TEXT NOT NULL, + SHORT_URL TEXT NOT NULL, + USERNAME TEXT ); """ with sqlite3.connect('db/urls.db') as conn: @@ -20,6 +21,6 @@ with sqlite3.connect('db/urls.db') as conn: conn = sqlite3.connect('db/urls.db') cursor = conn.cursor() -res = cursor.execute('SELECT LONG_URL, SHORT_URL FROM WEB_URL WHERE 1') #read all data from database +res = cursor.execute('SELECT LONG_URL, SHORT_URL, USERNAME FROM WEB_URL WHERE 1') #read all data from database for entries in res.fetchall(): - print(str(entries[1]).replace('"', "") + ";" + str(entries[0]).replace('"', "")) #format the data and print it to console. with a pipe you can redirect the output to a file e.g. python export.py > urls.csv \ No newline at end of file + print(str(entries[1]).replace('"', "") + ";" + str(entries[0]).replace('"', "") + ";" + entries[2]) #format the data and print it to console. with a pipe you can redirect the output to a file e.g. python export.py > urls.csv \ No newline at end of file diff --git a/import.py b/import.py index 7147eaf7ae9425bbf2aa179ca62fbef120ddd5c4..e22960dc9923c6bf54a943ed3bd3e198fa2832e4 100644 --- a/import.py +++ b/import.py @@ -9,7 +9,8 @@ def table_check(): #Check if database exists create_table = """ CREATE TABLE WEB_URL( LONG_URL TEXT NOT NULL, - SHORT_URL TEXT NOT NULL + SHORT_URL TEXT NOT NULL, + USERNAME TEXT ); """ with sqlite3.connect('db/urls.db') as conn: @@ -35,7 +36,8 @@ with sqlite3.connect('db/urls.db') as conn: for lines in tqdm(text): SHORT_URL = lines.split(";")[0].replace("\n", "").replace("\r","") #Split the CSV at the ";" then use the first one and replace all linebreaks LONG_URL = lines.split(";")[1].replace("\n", "").replace("\r","") #Split the CSV at the ";" then use the seccond one and replace all linebreaks + USERNAME = lines.split(";")[2].replace("\n", "").replace("\r","") #Split the CSV at the ";" then use the thirs one and replace all linebreaks res = cursor.execute( #Insert the data in the SQL table - 'INSERT INTO WEB_URL (LONG_URL, SHORT_URL) VALUES (?, ?)', - [LONG_URL, SHORT_URL.lower()] #replace big letters in short url with small letters. + 'INSERT INTO WEB_URL (LONG_URL, SHORT_URL, USERNAME) VALUES (?, ?, ?)', + [LONG_URL, SHORT_URL.lower(), USERNAME] #replace big letters in short url with small letters. ) \ No newline at end of file diff --git a/main.py b/main.py index 9e57fdf076415b3f5c3db67fa45e597ce2821659..f16419eeec636b9f754a70fcead5b89734b5cf85 100644 --- a/main.py +++ b/main.py @@ -1,15 +1,18 @@ #!/usr/bin/env python3 from waitress import serve #Used as webserver (Production) -from flask import Flask, request, render_template, redirect, abort, Markup #Used to prepare the dynamic pages (The main site) +from flask import Flask, request, render_template, redirect, abort, Markup, session, make_response #Used to prepare the dynamic pages (The main site) import sqlite3 #Used to store the Data import os #Used for getting the enviorement variables import qrcode #Used to generate the QR import base64 #Used to encode the generated QR as base64, to directly insert it into the HTML -from requests import post #Used to validate recaptcha +from requests import post, get #Used to validate recaptcha / oauth from io import BytesIO #Needed for base64 encoding of the image from PIL import Image #Needed for QR generation - +from flask_github import GitHub #github oauth library +import json #used for github oauth +from html import escape #This is used to escape characters, if they are send in the url app = Flask(__name__) + domain_to_index = {} try: @@ -64,6 +67,14 @@ try: except: host="127.0.0.1" +try: + app.config['GITHUB_CLIENT_ID'] = os.environ['GITHUB_CLIENT_ID'] + app.config['GITHUB_CLIENT_SECRET'] = os.environ['GITHUB_CLIENT_SECRET'] +except: + print("github client id sor client secret is not set, please set these and run again.") + exit() +github = GitHub(app) + index = 0 domain_prepared = "" for domains in domain: #Make from every domnain a entry for the select box later @@ -81,7 +92,8 @@ def table_check(): #This function is used on start to make a new Database if not create_table = """ CREATE TABLE WEB_URL( LONG_URL TEXT NOT NULL, - SHORT_URL TEXT NOT NULL + SHORT_URL TEXT NOT NULL, + USERNAME TEXT ); """ with sqlite3.connect('db/urls.db') as conn: @@ -119,21 +131,33 @@ def grecaptcha_verify(request): #This function is used to verify the google reca return response + @app.route('/', methods=['GET']) def home_get(): - return render_template('home.html', builddate=builddate, version=version, domain=domain_prepared, recaptchaPublicKey=recaptchaPublicKey, showDomainSelect=showDomainSelect) #return the default site to create a new shorten link + try: + loginbar = "Hello " + request.cookies.get('username') + ' (<a href="/user/links" style="color:white">your links</a>, <a href="/user/logout" style="color:white">logout</a>)' + except: + loginbar = '<a href="/user/login" style="color:white">login</a>' + + return render_template('home.html', builddate=builddate, version=version, domain=domain_prepared, recaptchaPublicKey=recaptchaPublicKey, showDomainSelect=showDomainSelect, loginbar=loginbar) #return the default site to create a new shorten link @app.route('/', methods=['POST']) #This function is used to create a new url def home_post(): + + try: + userID = request.cookies.get('userID') + loginbar = "Hello " + request.cookies.get('username') + ' (<a href="/user/links" style="color:white">your links</a>, <a href="/user/logout" style="color:white">logout</a>)' + except: + userID = "null" + loginbar = '<a href="/user/login" style="color:white">login</a>' if not grecaptcha_verify(request) and not skipCaptcha: - return render_template('home.html', builddate=builddate, version=version, domain=domain_prepared, snackbar="There was an error validating, that you are a human, please try again.", long_url_prefilled=request.form.get('url'), short_url_prefilled=request.form.get('short').lower(), domain_prefilled=domain_to_index[request.form.get('domain')], recaptchaPublicKey=recaptchaPublicKey, showDomainSelect=showDomainSelect) #return the user the prefilled form with an error message, because no url to short was provided - - + return render_template('home.html', builddate=builddate, version=version, domain=domain_prepared, snackbar="There was an error validating, that you are a human, please try again.", long_url_prefilled=request.form.get('url'), short_url_prefilled=request.form.get('short').lower(), domain_prefilled=domain_to_index[request.form.get('domain')], recaptchaPublicKey=recaptchaPublicKey, showDomainSelect=showDomainSelect, loginbar=loginbar) #return the user the prefilled form with an error message, because no url to short was provided + if (request.form.get('url').replace(" ", "") == ""): - return render_template('home.html', builddate=builddate, version=version, domain=domain_prepared, snackbar="Please enter a url to short, before submitting this form", long_url_prefilled=request.form.get('url'), short_url_prefilled=request.form.get('short').lower(), domain_prefilled=domain_to_index[request.form.get('domain')], recaptchaPublicKey=recaptchaPublicKey, showDomainSelect=showDomainSelect) #return the user the prefilled form with an error message, because no url to short was provided + return render_template('home.html', builddate=builddate, version=version, domain=domain_prepared, snackbar="Please enter a url to short, before submitting this form", long_url_prefilled=request.form.get('url'), short_url_prefilled=request.form.get('short').lower(), domain_prefilled=domain_to_index[request.form.get('domain')], recaptchaPublicKey=recaptchaPublicKey, showDomainSelect=showDomainSelect, loginbar=loginbar) #return the user the prefilled form with an error message, because no url to short was provided if (request.form.get('short').replace(" ", "") == ""): - return render_template('home.html', builddate=builddate, version=version, domain=domain_prepared, snackbar="Please enter a short name, before submitting this form", long_url_prefilled=request.form.get('url'), short_url_prefilled=request.form.get('short').lower(), domain_prefilled=domain_to_index[request.form.get('domain')], recaptchaPublicKey=recaptchaPublicKey, showDomainSelect=showDomainSelect) #return the user the prefilled form with an error message, because no short link was provided + return render_template('home.html', builddate=builddate, version=version, domain=domain_prepared, snackbar="Please enter a short name, before submitting this form", long_url_prefilled=request.form.get('url'), short_url_prefilled=request.form.get('short').lower(), domain_prefilled=domain_to_index[request.form.get('domain')], recaptchaPublicKey=recaptchaPublicKey, showDomainSelect=showDomainSelect, loginbar=loginbar) #return the user the prefilled form with an error message, because no short link was provided shorturl = (request.form.get('domain') + "/" + request.form.get('short')).lower() url = request.form.get('url') @@ -150,12 +174,12 @@ def home_post(): if not already_used: #If short link wasn't used before, insert the link in the Database. res = cursor.execute( - 'INSERT INTO WEB_URL (LONG_URL, SHORT_URL) VALUES (?, ?)', - [url, shorturl] + 'INSERT INTO WEB_URL (LONG_URL, SHORT_URL, USERNAME) VALUES (?, ?, ?)', + [url, shorturl, userID] ) - return render_template('home.html', short_url=shorturl, recaptchaPublicKey=recaptchaPublicKey, builddate=builddate, version=version, domain=domain_prepared, qrcode=makeQR("http://" + shorturl)) #return the shorten link to the user + return render_template('home.html', short_url=shorturl, recaptchaPublicKey=recaptchaPublicKey, builddate=builddate, version=version, domain=domain_prepared, qrcode=makeQR("http://" + shorturl), loginbar=loginbar) #return the shorten link to the user else: - return render_template('home.html', builddate=builddate, version=version, recaptchaPublicKey=recaptchaPublicKey, domain=domain_prepared, snackbar="URL already used, please try another one", long_url_prefilled=request.form.get('url'), short_url_prefilled=request.form.get('short').lower(), domain_prefilled=domain_to_index[request.form.get('domain')], showDomainSelect=showDomainSelect) #return the user the prefilled form with an error message, because the url was already used + return render_template('home.html', builddate=builddate, version=version, recaptchaPublicKey=recaptchaPublicKey, domain=domain_prepared, snackbar="URL already used, please try another one", long_url_prefilled=request.form.get('url'), short_url_prefilled=request.form.get('short').lower(), domain_prefilled=domain_to_index[request.form.get('domain')], showDomainSelect=showDomainSelect, loginbar=loginbar) #return the user the prefilled form with an error message, because the url was already used @app.route('/favicon.ico') #Redirect to the static url of the favicon def favicon(): @@ -184,6 +208,75 @@ def redirect_short_url(short_url): abort(404) +@app.route('/user/login') +def login(): + return github.authorize(scope="user") + +@app.route('/user/github-callback') +@github.authorized_handler +def authorized(oauth_token): + if oauth_token is None: + return "oauth failed, please try again" + + headers = {'Authorization': 'token ' + oauth_token,} #Useragent doesn't matters, but is set here + githubResponse = get("https://api.github.com/user", headers=headers).text + userID = str(json.loads(githubResponse)['id']) + username = str(json.loads(githubResponse)['login']) + + resp = make_response(redirect('/')) + resp.set_cookie('userID', userID) + resp.set_cookie('username', username) + return resp + +@app.route('/user/logout') +def logout(): + resp = make_response("logout successful") + resp.set_cookie('userID', "", max_age=0) + resp.set_cookie('username', "", max_age=0) + return resp + +@app.route('/user/links') +def ownLinks(): + try: + userID = request.cookies.get('userID') + loginbar = "Hello " + request.cookies.get('username') + ' (<a href="/user/logout" style="color:white">logout</a>)' + except: + return redirect("/user/login") + + with sqlite3.connect('db/urls.db') as conn: #Get the original URL from the database + cursor = conn.cursor() + res = cursor.execute('SELECT LONG_URL, SHORT_URL FROM WEB_URL WHERE USERNAME=?', [userID]) + response = '<table id="t01">\n<tr>\n<th>Long URL</th>\n<th>Short URL</th>\n<th>Action</th>\n</tr>\n' + try: + entriesList = res.fetchall() + for entries in entriesList: + response = response + "<tr>\n<td>" + entries[0] + "</td>\n<td>" + entries[1] + '</td>\n<td><a id="red" href="/user/delete?link=' + escape(entries[1]) + '">delete</a></tr>\n' + + if(len(entriesList) == 0): response = 'you have no shorten links. <a href="/">back</a>' + except: + abort(500) + response = response + "</table>" + return render_template('editEntries.html', content=response, loginbar=loginbar) + + +@app.route('/user/delete') +def delete(): + try: + userID = request.cookies.get('userID') + loginbar = "Hello " + request.cookies.get('username') + ' (<a href="/user/logout" style="color:white">logout</a>)' + except: + return redirect("/user/login") + linkToDelete = request.args.get('link') + + with sqlite3.connect('db/urls.db') as conn: #Get the original URL from the database + cursor = conn.cursor() + try: + cursor.execute('DELETE FROM WEB_URL WHERE SHORT_URL=? AND USERNAME=?', [linkToDelete, userID]) + return redirect('/user/links') + except: + abort(500) + + if __name__ == '__main__': table_check()# This code checks whether database table is created or not if production: #Check if production variable is set to true use the waitress webserver, else use the buildin flask webserver, with more debug output diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..fd268e603018c527fdc790a5f0e970ec4bb3e12a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +waitress +flask +qrcode +requests +Pillow +GitHub-Flask +tqdm \ No newline at end of file diff --git a/static/style.css b/static/style.css index 42bb370f0f9b5c53d5e111a74715011481f2cf62..bedcc8ff4cf824e8e5b006370b668b4033e2bba3 100644 --- a/static/style.css +++ b/static/style.css @@ -64,11 +64,11 @@ margin: auto; position: relative; z-index: 1; background: #FFFFFF; -max-width: 360px; margin: 0 auto 100px; padding: 45px; text-align: center; box-shadow: 0 0 20px 0 rgba(0, 0, 0, 0.2), 0 5px 5px 0 rgba(0, 0, 0, 0.24); +display: table; } .form input { font-family: "Roboto", sans-serif; @@ -176,6 +176,41 @@ visibility: visible; animation: fadein 0.5s, fadeout 0.5s 2.5s; } +#loginbar { + min-width: 250px; + background-color: #333; + color: #fff; + text-align: center; + border-radius: 2px; + padding: 16px; + position: fixed; + z-index: 1; + top: 30px; + right: 16px; + font-size: 17px; + } + +table, th, td { + border: 1px solid black; + border-collapse: collapse; +} +th, td { + padding: 15px; + text-align: left; +} +table#t01 tr:nth-child(even) { + background-color: #eee; +} +table#t01 tr:nth-child(odd) { + background-color: #fff; +} +table#t01 th { + background-color: black; + color: white; +} +a#red { + color: red; +} @-webkit-keyframes fadein { from {bottom: 0; opacity: 0;} to {bottom: 30px; opacity: 1;} diff --git a/templates/editEntries.html b/templates/editEntries.html new file mode 100644 index 0000000000000000000000000000000000000000..c460bdfb748cd1857c877a0d282f58161bb8f35f --- /dev/null +++ b/templates/editEntries.html @@ -0,0 +1,16 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <link href="{{ url_for('static', filename='style.css') }}" rel="stylesheet"> + <title>URL shorter</title> + </head> + + <body> + <div id="loginbar">{{loginbar | safe}}</div> + <div class="login-page"> + <div class="form"> + {{content | safe}} + </div> + </div> + </body> +</html> \ No newline at end of file diff --git a/templates/home.html b/templates/home.html index dba4e22b858c0076d234fb07d2a4948899d89fa1..4145407bfd4584af27c5a8b4366a7dcf54922e5c 100644 --- a/templates/home.html +++ b/templates/home.html @@ -3,7 +3,7 @@ <head> <link href="{{ url_for('static', filename='style.css') }}" rel="stylesheet"> <title>URL shorter</title> - {% if recaptchaPrivateKey %} + {% if recaptchaPublicKey %} <script src="https://www.google.com/recaptcha/api.js" async defer></script> <script> function onSubmit(token) { @@ -14,6 +14,7 @@ </head> <body> + <div id="loginbar">{{loginbar | safe}}</div> <div class="login-page"> <div class="form"> {% if not short_url %} @@ -29,7 +30,7 @@ {% endif %} <input id="short" name="short" type="text" placeholder="short name" value="{{short_url_prefilled}}"/> - {% if recaptchaPrivateKey %} + {% if recaptchaPublicKey %} <button class="g-recaptcha" data-sitekey="{{recaptchaPublicKey}}" data-callback='onSubmit'>short</button> {% else %} <button>short</button>