Merge pull request #2 from nexy7574/feature/logging

Feature/logging
This commit is contained in:
Nexus 2023-12-05 18:07:08 +00:00 committed by GitHub
commit 711f666d8c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 129 additions and 326 deletions

View file

@ -90,25 +90,11 @@ class Events(commands.Cog):
self.bot.bridge_queue = asyncio.Queue()
self.fetch_discord_atom_feed.start()
self.bridge_health = False
self.log = logging.getLogger("jimmy.cogs.events")
def cog_unload(self):
self.fetch_discord_atom_feed.cancel()
# noinspection DuplicatedCode
async def analyse_text(self, text: str) -> Optional[Tuple[float, float, float, float]]:
"""Analyse text for positivity, negativity and neutrality."""
def inner():
try:
from utils.sentiment_analysis import intensity_analyser
except ImportError:
return None
scores = intensity_analyser.polarity_scores(text)
return scores["pos"], scores["neu"], scores["neg"], scores["compound"]
async with self.bot.training_lock:
return await self.bot.loop.run_in_executor(None, inner)
@commands.Cog.listener("on_raw_reaction_add")
async def on_raw_reaction_add(self, payload: discord.RawReactionActionEvent):
channel: Optional[discord.TextChannel] = self.bot.get_channel(payload.channel_id)
@ -126,16 +112,17 @@ class Events(commands.Cog):
if member.guild is None or member.guild.id not in guilds:
return
student: Optional[Student] = await get_or_none(Student, user_id=member.id)
if student and student.id:
role = discord.utils.find(lambda r: r.name.lower() == "verified", member.guild.roles)
if role and role < member.guild.me.top_role:
await member.add_roles(role, reason="Verified")
# student: Optional[Student] = await get_or_none(Student, user_id=member.id)
# if student and student.id:
# role = discord.utils.find(lambda r: r.name.lower() == "verified", member.guild.roles)
# if role and role < member.guild.me.top_role:
# await member.add_roles(role, reason="Verified")
channel: discord.TextChannel = discord.utils.get(member.guild.text_channels, name="general")
if channel and channel.can_send():
await channel.send(
f"{LTR} {member.mention} (`{member}`, {f'{student.id}' if student else 'pending verification'})"
# f"{LTR} {member.mention} (`{member}`, {f'{student.id}' if student else 'pending verification'})"
f"{LTR} {member.mention}"
)
@commands.Cog.listener()
@ -143,11 +130,12 @@ class Events(commands.Cog):
if member.guild is None or member.guild.id not in guilds:
return
student: Optional[Student] = await get_or_none(Student, user_id=member.id)
# student: Optional[Student] = await get_or_none(Student, user_id=member.id)
channel: discord.TextChannel = discord.utils.get(member.guild.text_channels, name="general")
if channel and channel.can_send():
await channel.send(
f"{RTL} {member.mention} (`{member}`, {f'{student.id}' if student else 'pending verification'})"
# f"{RTL} {member.mention} (`{member}`, {f'{student.id}' if student else 'pending verification'})"
f"{RTL} {member.mention}"
)
async def process_message_for_github_links(self, message: discord.Message):
@ -248,7 +236,7 @@ class Events(commands.Cog):
)
region = message.author.voice.channel.rtc_region
# noinspection PyUnresolvedReferences
console.log(
self.log.warning(
"Timed out connecting to voice channel: {0.name} in {0.guild.name} "
"(region {1})".format(
message.author.voice.channel, region.name if region else "auto (unknown)"
@ -269,7 +257,7 @@ class Events(commands.Cog):
_dc(voice),
)
if err is not None:
console.log(f"Error playing audio: {err}")
self.log.error(f"Error playing audio: {err}", exc_info=err)
self.bot.loop.create_task(message.add_reaction("\N{speaker with cancellation stroke}"))
else:
self.bot.loop.create_task(
@ -590,14 +578,14 @@ class Events(commands.Cog):
try:
response = await self.http.get("https://discordstatus.com/history.atom", headers=headers)
except httpx.HTTPError as e:
console.log("Failed to fetch discord atom feed:", e)
self.log.error("Failed to fetch discord atom feed: %r", e, exc_info=e)
return
if response.status_code == 304:
return
if response.status_code != 200:
console.log("Failed to fetch discord atom feed:", response.status_code)
self.log.error("Failed to fetch discord atom feed: HTTP/%s", response.status_code)
return
with file.open("wb") as f:

View file

@ -1,3 +1,5 @@
import logging
import discord
import httpx
from discord.ext import commands
@ -129,4 +131,4 @@ def setup(bot):
if OAUTH_REDIRECT_URI and OAUTH_ID:
bot.add_cog(InfoCog(bot))
else:
print("OAUTH_REDIRECT_URI not set, not loading info cog")
logging.getLogger("jimmy.cogs.info").warning("OAUTH_REDIRECT_URI not set, not loading info cog")

View file

@ -3,6 +3,7 @@ import fnmatch
import functools
import glob
import io
import logging
import pathlib
import openai
@ -69,31 +70,11 @@ try:
VOICES = [x.id for x in _engine.getProperty("voices")]
del _engine
except Exception as _pyttsx3_err:
print("Failed to load pyttsx3: %s" % _pyttsx3_err, file=sys.stderr)
logging.error("Failed to load pyttsx3: %r", _pyttsx3_err, exc_info=True)
pyttsx3 = None
VOICES = []
# class OllamaStreamReader:
# def __init__(self, response: httpx.Response):
# self.response = response
# self.stream = response.aiter_bytes(1)
# self._buffer = b""
#
# async def __aiter__(self):
# return self
#
# async def __anext__(self) -> dict[str, str | int | bool]:
# if self.response.is_stream_consumed:
# raise StopAsyncIteration
# self._buffer = b""
# while not self._buffer.endswith(b"}\n"):
# async for char in self.stream:
# self._buffer += char
#
# return json.loads(self._buffer.decode("utf-8", "replace"))
async def ollama_stream_reader(response: httpx.Response) -> typing.AsyncGenerator[
dict[str, str | int | bool], None
]:
@ -103,7 +84,7 @@ async def ollama_stream_reader(response: httpx.Response) -> typing.AsyncGenerato
loaded = json.loads(chunk)
yield loaded
except json.JSONDecodeError as e:
print("Failed to decode chunk %r: %r" % (chunk, e), file=sys.stderr)
logging.warning("Failed to decode chunk %r: %r", chunk, e)
pass
@ -141,6 +122,7 @@ class OtherCog(commands.Cog):
self.ollama_locks: dict[discord.Message, asyncio.Event] = {}
self.context_cache: dict[str, list[int]] = {}
self.log = logging.getLogger("jimmy.cogs.other")
def cog_unload(self):
self._worker_task.cancel()
@ -257,7 +239,7 @@ class OtherCog(commands.Cog):
return driver, driver_path
driver, driver_path = find_driver()
console.log(
self.log.info(
"Using driver '{}' with binary '{}' to screenshot '{}', as requested by {}.".format(
driver, driver_path, website, ctx.user
)
@ -295,7 +277,7 @@ class OtherCog(commands.Cog):
start_init = time()
driver, friendly_url = await asyncio.to_thread(_setup)
end_init = time()
console.log("Driver '{}' initialised in {} seconds.".format(driver_name, round(end_init - start_init, 2)))
self.log.info("Driver '{}' initialised in {} seconds.".format(driver_name, round(end_init - start_init, 2)))
def _edit(content: str):
self.bot.loop.create_task(ctx.interaction.edit_original_response(content=content))
@ -349,20 +331,6 @@ class OtherCog(commands.Cog):
)
return result
async def analyse_text(self, text: str) -> Optional[Tuple[float, float, float, float]]:
"""Analyse text for positivity, negativity and neutrality."""
def inner():
try:
from utils.sentiment_analysis import intensity_analyser
except ImportError:
return None
scores = intensity_analyser.polarity_scores(text)
return scores["pos"], scores["neu"], scores["neg"], scores["compound"]
async with self.bot.training_lock:
return await self.bot.loop.run_in_executor(None, inner)
@staticmethod
async def get_xkcd(session: aiohttp.ClientSession, n: int) -> dict | None:
async with session.get("https://xkcd.com/{!s}/info.0.json".format(n)) as response:
@ -445,40 +413,6 @@ class OtherCog(commands.Cog):
view = self.XKCDGalleryView(number)
return await ctx.respond(embed=embed, view=view)
@commands.slash_command()
async def sentiment(self, ctx: discord.ApplicationContext, *, text: str):
"""Attempts to detect a text's tone"""
await ctx.defer()
if not text:
return await ctx.respond("You need to provide some text to analyse.")
result = await self.analyse_text(text)
if result is None:
return await ctx.edit(content="Failed to load sentiment analysis module.")
embed = discord.Embed(title="Sentiment Analysis", color=discord.Colour.embed_background())
embed.add_field(name="Positive", value="{:.2%}".format(result[0]))
embed.add_field(name="Neutral", value="{:.2%}".format(result[2]))
embed.add_field(name="Negative", value="{:.2%}".format(result[1]))
embed.add_field(name="Compound", value="{:.2%}".format(result[3]))
return await ctx.edit(content=None, embed=embed)
@commands.message_command(name="Detect Sentiment")
async def message_sentiment(self, ctx: discord.ApplicationContext, message: discord.Message):
await ctx.defer()
text = str(message.clean_content)
if not text:
return await ctx.respond("You need to provide some text to analyse.")
await ctx.respond("Analyzing (this may take some time)...")
result = await self.analyse_text(text)
if result is None:
return await ctx.edit(content="Failed to load sentiment analysis module.")
embed = discord.Embed(title="Sentiment Analysis", color=discord.Colour.embed_background())
embed.add_field(name="Positive", value="{:.2%}".format(result[0]))
embed.add_field(name="Neutral", value="{:.2%}".format(result[2]))
embed.add_field(name="Negative", value="{:.2%}".format(result[1]))
embed.add_field(name="Compound", value="{:.2%}".format(result[3]))
embed.url = message.jump_url
return await ctx.edit(content=None, embed=embed)
corrupt_file = discord.SlashCommandGroup(
name="corrupt-file",
description="Corrupts files.",
@ -1892,10 +1826,10 @@ class OtherCog(commands.Cog):
try:
model, tag = model.split(":", 1)
model = model + ":" + tag
print("Model %r already has a tag")
self.log.debug("Model %r already has a tag")
except ValueError:
model = model + ":latest"
print("Resolved model to %r" % model)
self.log.debug("Resolved model to %r" % model)
servers: dict[str, dict[str, str, list[str] | int]] = {
"100.106.34.86:11434": {
@ -1941,7 +1875,7 @@ class OtherCog(commands.Cog):
return True
for pat in _srv.get("allow", ['*']):
if not fnmatch.fnmatch(model_name.lower(), pat.lower()):
print(
self.log.debug(
"Server %r does not support %r (only %r.)" % (
_srv['name'],
model_name,

View file

@ -1,4 +1,5 @@
import json
import logging
import random
from datetime import datetime, time, timedelta, timezone
from pathlib import Path
@ -22,6 +23,7 @@ def schedule_times():
class TimeTableCog(commands.Cog):
def __init__(self, bot: commands.Bot):
self.bot = bot
self.log = logging.getLogger("jimmy.cogs.timetable")
with (Path.cwd() / "utils" / "timetable.json").open() as file:
self.timetable = json.load(file)
self.update_status.start()
@ -159,7 +161,7 @@ class TimeTableCog(commands.Cog):
try:
next_lesson = self.absolute_next_lesson(date + timedelta(days=1))
except RuntimeError:
print("Failed to fetch absolute next lesson. Is this the end?")
self.log.critical("Failed to fetch absolute next lesson. Is this the end?")
return
next_lesson.setdefault("name", "unknown")
next_lesson.setdefault("tutor", "unknown")

View file

@ -2,6 +2,7 @@ import asyncio
import datetime
import hashlib
import json
import logging
import random
import time
from datetime import timedelta
@ -66,15 +67,17 @@ class UptimeCompetition(commands.Cog):
def __init__(self, bot: commands.Bot):
self.bot = bot
self.http = AsyncClient(verify=False)
self.log = logging.getLogger("jimmy.cogs.uptime")
self.http = AsyncClient(verify=False, http2=True)
self._warning_posted = False
self.test_uptimes.add_exception_type(Exception)
self.test_uptimes.start()
self.last_result: list[UptimeEntry] = []
self.task_lock = asyncio.Lock()
self.task_event = asyncio.Event()
if not (pth := Path("targets.json")).exists():
pth.write_text(BASE_JSON)
self.path = Path.home() / ".cache" / "lcc-bot" / "targets.json"
if not self.path.exists():
self.path.write_text(BASE_JSON)
self._cached_targets = self.read_targets()
@property
@ -103,7 +106,7 @@ class UptimeCompetition(commands.Cog):
return response
def read_targets(self) -> List[Dict[str, str]]:
with open("targets.json") as f:
with self.path.open() as f:
data: list = json.load(f)
data.sort(key=lambda x: x["name"])
self._cached_targets = data.copy()
@ -111,7 +114,7 @@ class UptimeCompetition(commands.Cog):
def write_targets(self, data: List[Dict[str, str]]):
self._cached_targets = data
with open("targets.json", "w") as f:
with self.path.open("w") as f:
json.dump(data, f, indent=4, default=str)
def cog_unload(self):
@ -214,7 +217,7 @@ class UptimeCompetition(commands.Cog):
okay_statuses = list(filter(None, okay_statuses))
guild: discord.Guild = self.bot.get_guild(guild_id)
if guild is None:
console.log(
self.log.warning(
f"[yellow]:warning: Unable to locate the guild for {target['name']!r}! Can't uptime check."
)
else:
@ -224,7 +227,7 @@ class UptimeCompetition(commands.Cog):
try:
user = await guild.fetch_member(user_id)
except discord.HTTPException:
console.log(f"[yellow]:warning: Unable to locate {target['name']!r}! Can't uptime check.")
self.log.warning(f"[yellow]Unable to locate {target['name']!r}! Can't uptime check.")
user = None
if user:
create_tasks.append(
@ -240,7 +243,7 @@ class UptimeCompetition(commands.Cog):
)
else:
if self._warning_posted is False:
console.log(
self.log.warning(
"[yellow]:warning: Jimmy does not have the presences intent enabled. Uptime monitoring of the"
" shronk bot is disabled."
)

59
main.py
View file

@ -1,8 +1,10 @@
import asyncio
import os
import signal
import sys
import logging
import textwrap
from datetime import datetime, timedelta, timezone
from rich.logging import RichHandler
import config
import discord
@ -12,17 +14,47 @@ from utils import JimmyBanException, JimmyBans, console, get_or_none
from utils.client import bot
logging.basicConfig(
filename="jimmy.log",
filemode="a",
format="%(asctime)s:%(level)s:%(name)s: %(message)s",
format="%(asctime)s:%(levelname)s:%(name)s: %(message)s",
datefmt="%Y-%m-%d:%H:%M",
level=logging.INFO
level=logging.DEBUG,
force=True,
handlers=[
RichHandler(
getattr(config, "LOG_LEVEL", logging.INFO),
console=console,
markup=True,
rich_tracebacks=True,
show_path=False,
show_time=False
),
logging.FileHandler(
"jimmy.log",
"a"
)
]
)
logging.getLogger("discord.gateway").setLevel(logging.WARNING)
for _ln in [
"discord.client",
"httpcore.connection",
"httpcore.http2",
"hpack.hpack",
"discord.http",
"discord.bot",
"httpcore.http11",
"aiosqlite",
"httpx"
]:
logging.getLogger(_ln).setLevel(logging.INFO)
if os.name != "nt":
signal.signal(signal.SIGTERM, lambda: bot.loop.run_until_complete(bot.close()))
@bot.listen()
async def on_connect():
console.log("[green]Connected to discord!")
bot.log.info("[green]Connected to discord!")
@bot.listen("on_application_command_error")
@ -66,7 +98,7 @@ async def on_command_error(ctx: commands.Context, error: Exception):
@bot.listen("on_application_command")
async def on_application_command(ctx: discord.ApplicationContext):
console.log(
bot.log.info(
"{0.author} ({0.author.id}) used application command /{0.command.qualified_name} in "
"[blue]#{0.channel}[/], {0.guild}".format(ctx)
)
@ -74,9 +106,9 @@ async def on_application_command(ctx: discord.ApplicationContext):
@bot.event
async def on_ready():
console.log("Logged in as", bot.user)
bot.log.info("(READY) Logged in as %r", bot.user)
if getattr(config, "CONNECT_MODE", None) == 1:
console.log("Bot is now ready and exit target 1 is set, shutting down.")
bot.log.critical("Bot is now ready and exit target 1 is set, shutting down.")
await bot.close()
sys.exit(0)
@ -105,10 +137,11 @@ async def check_not_banned(ctx: discord.ApplicationContext | commands.Context):
if __name__ == "__main__":
console.log("Starting...")
bot.log.info("Starting...")
bot.started_at = discord.utils.utcnow()
if getattr(config, "WEB_SERVER", True):
bot.log.info("Web server is enabled (WEB_SERVER=True in config.py), initialising.")
import uvicorn
from web.server import app
@ -119,11 +152,11 @@ if __name__ == "__main__":
app,
host=getattr(config, "HTTP_HOST", "127.0.0.1"),
port=getattr(config, "HTTP_PORT", 3762),
loop="asyncio",
**getattr(config, "UVICORN_CONFIG", {}),
)
bot.log.info("Web server will listen on %s:%s", http_config.host, http_config.port)
server = uvicorn.Server(http_config)
console.log("Starting web server...")
bot.log.info("Starting web server...")
loop = bot.loop
http_server_task = loop.create_task(server.serve())
bot.web = {
@ -131,5 +164,5 @@ if __name__ == "__main__":
"config": http_config,
"task": http_server_task,
}
bot.log.info("Beginning main loop.")
bot.run(config.token)

View file

@ -4,7 +4,6 @@ from urllib.parse import urlparse
from discord.ext import commands
from ._email import *
from .console import *
from .db import *
from .views import *

View file

@ -1,52 +0,0 @@
import secrets
from email.message import EmailMessage
import aiosmtplib as smtp
import config
import discord
gmail_cfg = {"addr": "smtp.gmail.com", "username": config.email, "password": config.email_password, "port": 465}
TOKEN_LENGTH = 16
class _FakeUser:
def __init__(self):
with open("/etc/dictionaries-common/words") as file:
names = file.readlines()
names = [x.strip() for x in names if not x.strip().endswith("'s")]
self.names = names
def __str__(self):
import random
return f"{random.choice(self.names)}#{str(random.randint(1, 9999)).zfill(4)}"
async def send_verification_code(user: discord.User, student_number: str, **kwargs) -> str:
"""Sends a verification code, returning said verification code, to the student."""
code = secrets.token_hex(TOKEN_LENGTH)
text = (
f"Hey {user} ({student_number})! The code to join Unscrupulous Nonsense is '{code}'.\n\n"
f"Go back to the #verify channel, and click 'I have a verification code!', and put {code} in the modal"
f" that pops up\n\n"
f"If you have any issues getting in, feel free to reply to this email, or DM eek#7574.\n"
f"~Nex\n\n\n"
f"(P.S you can now go to http://droplet.nexy7574.co.uk/jimmy/verify/{code} instead)"
)
msg = EmailMessage()
msg["From"] = msg["bcc"] = "B593764@my.leedscitycollege.ac.uk"
msg["To"] = f"{student_number}@my.leedscitycollege.ac.uk"
msg["Subject"] = "Server Verification"
msg.set_content(text)
kwargs.setdefault("hostname", gmail_cfg["addr"])
kwargs.setdefault("port", gmail_cfg["port"])
kwargs.setdefault("use_tls", True)
kwargs.setdefault("username", gmail_cfg["username"])
kwargs.setdefault("password", gmail_cfg["password"])
kwargs.setdefault("start_tls", not kwargs["use_tls"])
assert kwargs["start_tls"] != kwargs["use_tls"]
await smtp.send(msg, **kwargs)
return code

View file

@ -1,4 +1,5 @@
import asyncio
import logging
import sys
from asyncio import Lock
from datetime import datetime, timezone
@ -30,37 +31,40 @@ class Bot(commands.Bot):
super().__init__(
command_prefix=commands.when_mentioned_or(*prefixes),
debug_guilds=guilds,
allowed_mentions=discord.AllowedMentions.none(),
allowed_mentions=discord.AllowedMentions(everyone=False, users=True, roles=False, replied_user=True),
intents=intents,
max_messages=5000,
case_insensitive=True,
)
self.loop.run_until_complete(registry.create_all())
self.training_lock = Lock()
self.started_at = datetime.now(tz=timezone.utc)
self.started_at = discord.utils.utcnow()
self.console = console
self.incidents = {}
self.log = log = logging.getLogger("jimmy.client")
self.debug = log.debug
self.info = log.info
self.warning = self.warn = log.warning
self.error = self.log.error
self.critical = self.log.critical
for ext in extensions:
try:
self.load_extension(ext)
except discord.ExtensionNotFound:
console.log(f"[red]Failed to load extension {ext}: Extension not found.")
log.error(f"[red]Failed to load extension {ext}: Extension not found.")
except (discord.ExtensionFailed, OSError) as e:
console.log(f"[red]Failed to load extension {ext}: {e}")
if getattr(config, "dev", False):
console.print_exception()
log.error(f"[red]Failed to load extension {ext}: {e}", exc_info=True)
else:
console.log(f"Loaded extension [green]{ext}")
log.info(f"Loaded extension [green]{ext}")
if getattr(config, "CONNECT_MODE", None) == 2:
async def connect(self, *, reconnect: bool = True) -> None:
self.console.log("Exit target 2 reached, shutting down (not connecting to discord).")
self.log.critical("Exit target 2 reached, shutting down (not connecting to discord).")
return
async def on_error(self, event: str, *args, **kwargs):
e_type, e, tb = sys.exc_info()
if isinstance(e, discord.NotFound) and e.code == 10062: # invalid interaction
self.log.warning(f"Invalid interaction received, ignoring. {e!r}")
return
if isinstance(e, discord.CheckFailure) and "The global check once functions failed." in str(e):
return
@ -69,25 +73,27 @@ class Bot(commands.Bot):
async def close(self) -> None:
await self.http.close()
if getattr(self, "web", None) is not None:
self.console.log("Closing web server...")
await self.web["server"].shutdown()
if hasattr(self, "web"):
self.log.info("Closing web server...")
try:
await asyncio.wait_for(self.web["server"].shutdown(), timeout=5)
self.web["task"].cancel()
self.console.log("Web server closed.")
try:
await self.web["task"]
except asyncio.CancelledError:
await asyncio.wait_for(self.web["task"], timeout=5)
except (asyncio.CancelledError, asyncio.TimeoutError):
pass
del self.web["server"]
del self.web["config"]
del self.web["task"]
del self.web
except asyncio.TimeoutError:
pass
try:
await super().close()
except asyncio.TimeoutError:
self.console.log("Timed out while closing, forcing shutdown.")
self.log.critical("Timed out while closing, forcing shutdown.")
sys.exit(1)
self.console.log("Finished shutting down.")
self.log.info("Finished shutting down.")
try:

View file

@ -1,112 +0,0 @@
# I have NO idea how this works
# I copied it from the tutorial
# However it works
import random
import re
import string
from nltk import FreqDist, NaiveBayesClassifier, classify
from nltk.corpus import movie_reviews, stopwords, twitter_samples
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.tag import pos_tag
positive_tweets = twitter_samples.strings("positive_tweets.json")
negative_tweets = twitter_samples.strings("negative_tweets.json")
positive_reviews = movie_reviews.categories("pos")
negative_reviews = movie_reviews.categories("neg")
positive_tweets += positive_reviews
# negative_tweets += negative_reviews
positive_tweet_tokens = twitter_samples.tokenized("positive_tweets.json")
negative_tweet_tokens = twitter_samples.tokenized("negative_tweets.json")
text = twitter_samples.strings("tweets.20150430-223406.json")
tweet_tokens = twitter_samples.tokenized("positive_tweets.json")
stop_words = stopwords.words("english")
def lemmatize_sentence(_tokens):
lemmatizer = WordNetLemmatizer()
lemmatized_sentence = []
for word, tag in pos_tag(_tokens):
if tag.startswith("NN"):
pos = "n"
elif tag.startswith("VB"):
pos = "v"
else:
pos = "a"
lemmatized_sentence.append(lemmatizer.lemmatize(word, pos))
return lemmatized_sentence
def remove_noise(_tweet_tokens, _stop_words=()):
cleaned_tokens = []
for token, tag in pos_tag(_tweet_tokens):
token = re.sub("https?://(?:[a-zA-Z]|[0-9]|[$-_@.&+#]|[!*(),]|" "%[0-9a-fA-F][0-9a-fA-F])+", "", token)
token = re.sub("(@[A-Za-z0-9_]+)", "", token)
if tag.startswith("NN"):
pos = "n"
elif tag.startswith("VB"):
pos = "v"
else:
pos = "a"
lemmatizer = WordNetLemmatizer()
token = lemmatizer.lemmatize(token, pos)
if len(token) > 0 and token not in string.punctuation and token.lower() not in _stop_words:
cleaned_tokens.append(token.lower())
return cleaned_tokens
positive_cleaned_tokens_list = []
negative_cleaned_tokens_list = []
for tokens in positive_tweet_tokens:
positive_cleaned_tokens_list.append(remove_noise(tokens, stop_words))
for tokens in negative_tweet_tokens:
negative_cleaned_tokens_list.append(remove_noise(tokens, stop_words))
def get_all_words(cleaned_tokens_list):
for _tokens in cleaned_tokens_list:
for token in _tokens:
yield token
all_pos_words = get_all_words(positive_cleaned_tokens_list)
freq_dist_pos = FreqDist(all_pos_words)
def get_tweets_for_model(cleaned_tokens_list):
for _tweet_tokens in cleaned_tokens_list:
yield {token: True for token in _tweet_tokens}
positive_tokens_for_model = get_tweets_for_model(positive_cleaned_tokens_list)
negative_tokens_for_model = get_tweets_for_model(negative_cleaned_tokens_list)
positive_dataset = [(tweet_dict, "Positive") for tweet_dict in positive_tokens_for_model]
negative_dataset = [(tweet_dict, "Negative") for tweet_dict in negative_tokens_for_model]
dataset = positive_dataset + negative_dataset
random.shuffle(dataset)
train_data = dataset[:7000]
test_data = dataset[7000:]
classifier = NaiveBayesClassifier.train(train_data)
intensity_analyser = SentimentIntensityAnalyzer()
if __name__ == "__main__":
while True:
try:
ex = input("> ")
except KeyboardInterrupt:
break
else:
print(classifier.classify({token: True for token in remove_noise(ex.split())}))
print(intensity_analyser.polarity_scores(ex))

View file

@ -9,14 +9,13 @@ import orm
from discord.ui import View
from utils import (
TOKEN_LENGTH,
BannedStudentID,
Student,
VerifyCode,
console,
get_or_none,
send_verification_code,
)
TOKEN_LENGTH = 16
if typing.TYPE_CHECKING:
from cogs.timetable import TimeTableCog
@ -133,7 +132,8 @@ class VerifyView(View):
)
try:
_code = await send_verification_code(interaction.user, st)
# _code = await send_verification_code(interaction.user, st)
raise RuntimeError("Disabled.")
except Exception as e:
return await interaction.followup.send(f"\N{cross mark} Failed to send email - {e}. Try again?")
console.log(f"Sending verification email to {interaction.user} ({interaction.user.id}/{st})...")

View file

@ -1,9 +1,8 @@
import asyncio
import ipaddress
import logging
import os
import sys
import textwrap
from rich import print
from datetime import datetime, timezone
from hashlib import sha512
from http import HTTPStatus
@ -37,7 +36,9 @@ try:
except ImportError:
WEB_ROOT_PATH = ""
GENERAL = "https://ptb.discord.com/channels/994710566612500550/1018915342317277215/"
log = logging.getLogger("jimmy.api")
GENERAL = "https://discord.com/channels/994710566612500550/"
OAUTH_ENABLED = OAUTH_ID and OAUTH_SECRET and OAUTH_REDIRECT_URI
@ -87,7 +88,7 @@ async def authenticate(req: Request, code: str = None, state: str = None):
if not (code and state) or state not in app.state.states:
value = os.urandom(4).hex()
if value in app.state.states:
print("Generated a state that already exists. Cleaning up", file=sys.stderr)
log.warning("Generated a state that already exists. Cleaning up")
# remove any states older than 5 minutes
removed = 0
for _value in list(app.state.states):
@ -95,24 +96,23 @@ async def authenticate(req: Request, code: str = None, state: str = None):
del app.state.states[_value]
removed += 1
value = os.urandom(4).hex()
print(f"Removed {removed} states.", file=sys.stderr)
log.warning(f"Removed {removed} old states.")
if value in app.state.states:
print("Critical: Generated a state that already exists and could not free any slots.", file=sys.stderr)
log.critical("Generated a state that already exists and could not free any slots.")
raise HTTPException(
HTTPStatus.SERVICE_UNAVAILABLE,
"Could not generate a state token (state container full, potential (D)DOS attack?). "
"Please try again later.",
# Saying a suspected DDOS makes sense, there are 4,294,967,296 possible states, the likelyhood of a
# collision is 1 in 4,294,967,296.
headers={"Retry-After": "300"},
headers={"Retry-After": "60"},
)
app.state.states[value] = datetime.now()
return RedirectResponse(
discord.utils.oauth_url(
OAUTH_ID, redirect_uri=OAUTH_REDIRECT_URI, scopes=("identify", "connections", "guilds", "email")
)
+ f"&state={value}&prompt=none",
) + f"&state={value}&prompt=none",
status_code=HTTPStatus.TEMPORARY_REDIRECT,
headers={"Cache-Control": "no-store, no-cache"},
)
@ -239,7 +239,7 @@ async def verify(code: str):
# And delete the code
await verify_code.delete()
console.log(f"[green]{verify_code.bind} verified ({verify_code.bind}/{verify_code.student_id})")
log.info(f"[green]{verify_code.bind} verified ({verify_code.bind}/{verify_code.student_id})")
return RedirectResponse(GENERAL, status_code=308)
@ -293,12 +293,12 @@ async def bridge(req: Request):
@app.websocket("/bridge/recv")
async def bridge_recv(ws: WebSocket, secret: str = Header(None)):
await ws.accept()
print("Websocket %r accepted." % ws)
log.info("Websocket %s:%s accepted.", ws.client.host, ws.client.port)
if secret != app.state.bot.http.token:
print("Closing websocket %r, invalid secret." % ws)
log.warning("Closing websocket %r, invalid secret.", ws.client.host)
raise _WSException(code=1008, reason="Invalid Secret")
if app.state.ws_connected.locked():
print("Closing websocket %r, already connected." % ws)
log.warning("Closing websocket %r, already connected." % ws)
raise _WSException(code=1008, reason="Already connected.")
queue: asyncio.Queue = app.state.bot.bridge_queue
@ -307,7 +307,7 @@ async def bridge_recv(ws: WebSocket, secret: str = Header(None)):
try:
await ws.send_json({"status": "ping"})
except (WebSocketDisconnect, WebSocketException):
print("Websocket %r disconnected." % ws)
log.info("Websocket %r disconnected.", ws)
break
try:
@ -316,10 +316,10 @@ async def bridge_recv(ws: WebSocket, secret: str = Header(None)):
continue
try:
print("Sent data %r to websocket %r." % (data, ws))
await ws.send_json(data)
log.debug("Sent data %r to websocket %r.", data, ws)
except (WebSocketDisconnect, WebSocketException):
print("Websocket %r disconnected." % ws)
log.info("Websocket %r disconnected." % ws)
break
finally:
queue.task_done()