properly migrate
This commit is contained in:
parent
7b09b75836
commit
0dc4117f10
GPG key ID:
0FA334385D0B689F
1 changed files with 97 additions and 72 deletions
169
server.py
169
server.py
|
|
@ -2,6 +2,7 @@ import contextlib
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import textwrap
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
|
|
@ -64,76 +65,84 @@ INVALID_TOKEN = JSONResponse(
|
||||||
401
|
401
|
||||||
)
|
)
|
||||||
VALID_OG_TAGS = [
|
VALID_OG_TAGS = [
|
||||||
"title",
|
"og:title",
|
||||||
"type",
|
"og:type",
|
||||||
"image",
|
"og:image",
|
||||||
"url",
|
"og:url",
|
||||||
"audio",
|
"og:audio",
|
||||||
"description",
|
"og:description",
|
||||||
"determiner",
|
"og:determiner",
|
||||||
"locale",
|
"og:locale",
|
||||||
"locale:alternative",
|
"og:locale:alternative",
|
||||||
"site_name",
|
"og:site_name",
|
||||||
"image:url",
|
"og:image:url",
|
||||||
"image:secure_url",
|
"og:image:secure_url",
|
||||||
"image:type",
|
"og:image:type",
|
||||||
"image:width",
|
"og:image:width",
|
||||||
"image:height",
|
"og:image:height",
|
||||||
"image:alt",
|
"og:image:alt",
|
||||||
"video",
|
"og:video",
|
||||||
"video:url",
|
"og:video:url",
|
||||||
"video:secure_url",
|
"og:video:secure_url",
|
||||||
"video:type",
|
"og:video:type",
|
||||||
"video:width",
|
"og:video:width",
|
||||||
"video:height",
|
"og:video:height",
|
||||||
"video:alt",
|
"og:video:alt",
|
||||||
"video:actor",
|
"og:video:actor",
|
||||||
"video:actor:role",
|
"og:video:actor:role",
|
||||||
"video:director",
|
"og:video:director",
|
||||||
"video:writer",
|
"og:video:writer",
|
||||||
"video:duration",
|
"og:video:duration",
|
||||||
"video:release_date",
|
"og:video:release_date",
|
||||||
"video:tag",
|
"og:video:tag",
|
||||||
"video:series"
|
"og:video:series"
|
||||||
"audio:url",
|
"og:audio:url",
|
||||||
"audio:secure_url",
|
"og:audio:secure_url",
|
||||||
"audio:type",
|
"og:audio:type",
|
||||||
"music:duration",
|
"og:music:duration",
|
||||||
"music:album",
|
"og:music:album",
|
||||||
"music:album:disc",
|
"og:music:album:disc",
|
||||||
"music:album:track",
|
"og:music:album:track",
|
||||||
"music:musician",
|
"og:music:musician",
|
||||||
"music:song",
|
"og:music:song",
|
||||||
"music:song:disc",
|
"og:music:song:disc",
|
||||||
"music:song:track",
|
"og:music:song:track",
|
||||||
"music:release_date",
|
"og:music:release_date",
|
||||||
"music:creator",
|
"og:music:creator",
|
||||||
"article:published_time",
|
"og:article:published_time",
|
||||||
"article:modified_time",
|
"og:article:modified_time",
|
||||||
"article:expiration_time",
|
"og:article:expiration_time",
|
||||||
"article:author",
|
"og:article:author",
|
||||||
"article:section",
|
"og:article:section",
|
||||||
"article:tag",
|
"og:article:tag",
|
||||||
"book:author",
|
"og:book:author",
|
||||||
"book:tag",
|
"og:book:tag",
|
||||||
"book:isbn",
|
"og:book:isbn",
|
||||||
"book:release_date",
|
"og:book:release_date",
|
||||||
"profile:first_name",
|
"og:profile:first_name",
|
||||||
"profile:last_name",
|
"og:profile:last_name",
|
||||||
"profile:username",
|
"og:profile:username",
|
||||||
"profile:gender"
|
"og:profile:gender"
|
||||||
]
|
]
|
||||||
URL_OG_TAGS = [
|
URL_OG_TAGS = [
|
||||||
"video",
|
"og:video",
|
||||||
"video:url",
|
"og:video:url",
|
||||||
"video:secure_url",
|
"og:video:secure_url",
|
||||||
"image",
|
"og:image",
|
||||||
"image:url",
|
"og:image:url",
|
||||||
"image:secure_url",
|
"og:image:secure_url",
|
||||||
"audio",
|
"og:audio",
|
||||||
"audio:url",
|
"og:audio:url",
|
||||||
"audio:secure_url"
|
"og:audio:secure_url"
|
||||||
]
|
]
|
||||||
|
TWITTER_MAPPING = {
|
||||||
|
"twitter:site": "site_name",
|
||||||
|
"twitter:creator": "site_name",
|
||||||
|
"twitter:image": "image",
|
||||||
|
"twitter:title": "title",
|
||||||
|
"twitter:image:width": "image:width",
|
||||||
|
"twitter:image:height": "image:height",
|
||||||
|
}
|
||||||
|
|
||||||
if Path.cwd() == Path("/app"):
|
if Path.cwd() == Path("/app"):
|
||||||
logging.info("Look to be running in a docker container. Cache will be stored in /app/cache.")
|
logging.info("Look to be running in a docker container. Cache will be stored in /app/cache.")
|
||||||
|
|
@ -240,14 +249,30 @@ def preview_url(
|
||||||
og_tags = {}
|
og_tags = {}
|
||||||
|
|
||||||
for tag in soup.find_all("meta"):
|
for tag in soup.find_all("meta"):
|
||||||
if tag.get("property", "").startswith("og:"):
|
logging.debug("Found meta tag: %r", tag)
|
||||||
tag_name = tag.get("property")[3:]
|
if tag.get("property", "").startswith(("og:", "twitter:")):
|
||||||
if tag_name in VALID_OG_TAGS:
|
logging.debug(
|
||||||
|
"Tag %r is an OG/Twitter tag, with property: %r",
|
||||||
|
textwrap.shorten(tag.get("content", "N/A"), 100)
|
||||||
|
)
|
||||||
|
tag_name = tag.get("property")
|
||||||
|
if tag_name in (*VALID_OG_TAGS, *TWITTER_MAPPING.keys()):
|
||||||
og_tags[tag_name] = tag.get("content")
|
og_tags[tag_name] = tag.get("content")
|
||||||
|
|
||||||
|
for tag in og_tags.keys():
|
||||||
|
if tag.startswith("twitter:"):
|
||||||
|
if tag in TWITTER_MAPPING:
|
||||||
|
og_tags[TWITTER_MAPPING[tag]] = og_tags.pop(tag)
|
||||||
|
logging.debug("Mapped twitter tag %r to og tag %r", tag, TWITTER_MAPPING[tag])
|
||||||
|
else:
|
||||||
|
logging.warning("Unrecognized Twitter tag: %r", tag)
|
||||||
|
og_tags.pop(tag, None)
|
||||||
|
|
||||||
for tag_name in URL_OG_TAGS:
|
for tag_name in URL_OG_TAGS:
|
||||||
if tag_name in og_tags:
|
if tag_name in og_tags:
|
||||||
|
logging.debug("Retrieving tag %r to see if it needs uploading to Matrix", tag_name)
|
||||||
_url = og_tags[tag_name]
|
_url = og_tags[tag_name]
|
||||||
|
logging.debug("%r = %r", tag_name, _url)
|
||||||
try:
|
try:
|
||||||
# noinspection PyArgumentList
|
# noinspection PyArgumentList
|
||||||
with httpx.stream(
|
with httpx.stream(
|
||||||
|
|
@ -295,9 +320,9 @@ def preview_url(
|
||||||
response_media.headers.get("content-type", "")
|
response_media.headers.get("content-type", "")
|
||||||
)
|
)
|
||||||
if upload_response:
|
if upload_response:
|
||||||
og_tags["original:" + tag_name] = og_tags[tag_name]
|
og_tags["original:" + tag_name.replace("og:", "")] = og_tags[tag_name]
|
||||||
og_tags[tag_name] = upload_response
|
og_tags[tag_name] = upload_response
|
||||||
if tag_name in ["image", "image:url", "image:secure_url"]:
|
if tag_name in ["og:image", "og:image:url", "og:image:secure_url"]:
|
||||||
_file.seek(0)
|
_file.seek(0)
|
||||||
og_tags["matrix:image:size"] = len(_file.getvalue())
|
og_tags["matrix:image:size"] = len(_file.getvalue())
|
||||||
logging.info("Uploaded media: %r" % _url)
|
logging.info("Uploaded media: %r" % _url)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue