Companion

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Companion
# 2024 Copyright © Robert APM Darin
# All rights reserved unconditionally.

# ***** IMPORTANT:

#   This program uses ADMINISTRATOR priviledges.

# Forum and thread support is a strange approach. Threads are treated like channels,
# with a few extra bits. Webhooks, slowdown mode, edits, even the way message are sent
# into the thread is effected. Activitieshave to be tested and is it is a thread, most
# of the time, you have to pull the parent channel. Webhooks and slowdown mode, in
# particular, do NOT have separate functionalities of a channel, even though discord
# treats them like a channel.

### *** At some point, I'm going to really have to rewrite this. The forum/thread detection
###     code is a stinking hot mess and scattered everywhere.

# TODO list:
#   Figure out how to handle channel config setting across diferent servers.

# Listed in order of when the service was added.

# added openai
# added ollama
# added together.ai
# added cohere

# --> Not in a specific order, add support for the following engines:

#   anthropic
#   DeepInfra           https://deepinfra.com/pricing
#   google.ai (genini)
#   huggingface
#   perplexity.ai
#   fireworks.ai
#   OctoML AI
#   Vertex AI
#   AI21labs
#   Deep Infra
#   Azure
#   Anyscale
#   Replicate
#   octo.ai
#   openrouter.ai

# Special USER commands:

#   %http       Read URLs, YouTube transcripts, and PDFs
#   %yttags     Get YouTube video tage, if there are any.
#   %Forget

# Developer/Admin only:

#   %PurgeRequests
#   %CheckBot

# Needed functionality

#   Purge memory that are older then X days automatically

# Wanted features:

#   Imposter detection - completed
#   Auto slow mode - completed
#   Anti raid/nuke
#   Ticket management
#   Anti nudity verification for images.
#   URL check (abuseIPDB) verifications.

# Disable 3rd party logging.
import warnings
warnings.filterwarnings('ignore')
import logging
logging.basicConfig(level=logging.CRITICAL)
logger=logging.getLogger('transformers')
logger.setLevel(logging.CRITICAL)
logger.handlers=[]

import sys
import os
import io
import copy
import itertools
import datetime
import time
import json
import random
import string
import concurrent.futures
import threading
import urllib.request
import requests
import re
import asyncio
import discord
import profanity_check as pc
import pdfplumber
import ollama
import tiktoken
import openai
import together
import cohere
import youtube_transcript_api
import imagehash
from discord.ext import commands, tasks
from transformers import AutoTokenizer
from googleapiclient.discovery import build
from PIL import Image
from io import BytesIO

# Active version

Version="0.0.0.0.850"

# The running name of the program. Must be global and NEVER changing.

RunningName=sys.argv[0]

# Persona base folder. This is where all personas are stored.

CompanionBase='/home/Companion'
CompanionStorage=f'{CompanionBase}/Personas'
MemoryStorage=f'{CompanionBase}/Servers/Memory'
LoggingStorage=f'{CompanionBase}/Servers/Logs'
ConfigStorage=f'{CompanionBase}/Servers/Config'

# For anagram solver
AnagramWordList="/home/Companion/AnagramSolver.txt"

# This is a list of domains that are scams, frauds, or malitiouc. It is exists, it is read and message that
# have links listed are removed. Requires the persona text file as well for responses.

CompanionScamURLS='/home/Companion/Companion.scam-urls'
CompanionAutoFilter='/home/Companion/Companion.autofilter'

# Global lock for managing sequential access to the AI api and file safety.

ResponseLock=threading.Lock()
ResponseTimeout=60

DeleteLock=threading.Lock()
DeleteTimeout=60

DisectLock=threading.Lock()
DisectTimeout=60

LoggingLock=threading.Lock()
LoggingTimeout=60

BabbleLock=threading.Lock()
BabbleTimeout=300

# Constants For auto slowmode. Needs to be dymanic in the future
SLOW_MODE_DURATION = 3  # Seconds for individual slow mode
SLOW_MODE_COOLDOWN = 307  # 5 minutes/7 secords cooldown for slow mode adjustments

# Dictionary to store the last slow mode change time for each channel
last_slow_mode_change = {}
# For counting active users per chanel
active_users={}

# This is required for the bot to work properly.

intents=discord.Intents.all()
intents.presences=True
intents.messages=True
intents.message_content=True
intents.members=True
intents.guilds=True

# Create a Discord client
client=discord.Client(intents=intents)

### Really need to make these files on disk and not global memory lists, for
### sharding/multi process management.

# List to store pending requests
request_list=[]
# List to stowre timed messages for deletion
delete_list=[]

###
### General file tools
###

# Cheap mkdir

def mkdir(fn):
    if not os.path.exists(fn):
        os.makedirs(fn,exist_ok=True)

# Helper function to download and load an image from a URL

def LoadImageURL(url):
    response=requests.get(url)
    img=Image.open(BytesIO(response.content))
    return img

# Read file into buffer

def ReadFile(fn,binary=False):
    if os.path.exists(fn):
        if binary:
            cf=open(fn,'rb')
            buffer=cf.read()
            cf.close()
        else:
            cf=open(fn,'r')
            buffer=cf.read().strip()
            cf.close()
    else:
        buffer=None
    return buffer

# Write file to disk

def WriteFile(fn,data):
    cf=open(fn,'w')
    cf.write(data)
    cf.close()

# Pick one line from a file as a response

def ReadFile2List(fname,ForceLower=False):
    # Something broke. Keep the responses in character
    responses=ReadFile(fname).strip().split('\n')
    while '' in responses:
        responses.remove('')
    if ForceLower==True:
        responses=[item.lower() for item in responses]
    return responses

# Pick a random response from a filew
# Something broke. Keep the responses in character

def PickRandomResponse(fname):
    responses=ReadFile2List(fname)
    selected_response=random.choice(responses)
    if selected_response.startswith('{[(*') and selected_response.endswith('*)]}'):
        buffer=ReadFile(selected_response[4:-4].strip()).strip()
        return buffer
    return selected_response

###
### Random support functions
###

# Check a string for numbers only.

def NumberOnly(s):
    # Replace common look-alikes with their numeric equivalents
    look_alike_replacements={
        'I': '1',  # Uppercase 'i' as '1'
        'l': '1',  # Lowercase 'L' as '1'
        'O': '0',  # Uppercase 'O' as '0'
    }

    # Replace look-alikes in the input string
    for look_alike, digit in look_alike_replacements.items():
        s=s.replace(look_alike, digit)

    s=s.strip().replace(' ','')

    # Define valid characters, including look-alikes and numeric equivalents
    valid_chars=set("0123456789.,")  # Regular digits, comma, and period
    look_alike_chars=set("٠١٢٣٤٥٦٧٨٩")  # Arabic-Indic digits
    full_width_digits=set("０１２３４５６７８９")  # Full-width digits

    # Combine all valid characters into one set
    valid_chars.update(look_alike_chars)
    valid_chars.update(full_width_digits)

    # Check each character in the string
    for char in s:
        if char not in valid_chars:
            return False

    # Basic number validation: ensure the string isn't just commas or periods
    if s.replace(",", "").replace(".", "").isdigit():
        return True

    return False

# Leet list deritives for trying to get someone's age.

lside=['ask','how','is','are','your','you','when','whens','what','whats','wut','wuts','was','tell','ur','u','r']
rside=['over','old','young','under','teen','tween','tweenie','born','date','year','yr','age','birth','birthed','birthdate','birthday','bday','bd','born']

# Building the leet derivitives. This was a royal pain in the ass, but by doing so, if any user trying to
# bypass the edit detection can be persumed to have malicious intent.

def BuildDerivitives(word):
    substitutions={
        'a': ['@','4'],
        'e': ['3'],
        'o': ['0'],
        'l': ['1', '|', '!', 'i'],
        'i': ['1', 'l', '|', '!', 'l'],
        's': ['z', '$', '5'],
        't': ['7', '+']
    }

    # Start the list
    dList=[ word ]

    # Forward in loop
    for x in range(len(word)):
        xword=list(word)
        if xword[x] in substitutions.keys():
            cList=substitutions[xword[x]]
            for y in range(len(cList)):
                xword[x]=cList[y]
                nword=''.join(xword)
                if nword not in dList:
                    dList.append(nword)

    # Forward reset at beginning
    xword=list(word)
    for x in range(len(word)):
        if xword[x] in substitutions.keys():
            cList=substitutions[xword[x]]
            for y in range(len(cList)):
                xword[x]=cList[y]
                nword=''.join(xword)
                if nword not in dList:
                    dList.append(nword)

    # Backwards
    xword=list(word)
    for x in range(len(word)-1,-1,-1):
        if xword[x] in substitutions.keys():
            cList=substitutions[xword[x]]
            for y in range(len(cList)):
                xword[x]=cList[y]
                nword=''.join(xword)
                if nword not in dList:
                    dList.append(nword)

    # return the list of words back to user
    return dList

def BuildLeetList(side):
    leetlist=[]

    for i in range(len(side)):
        leet=BuildDerivitives(side[i])
        for j in range(len(leet)):
            if leet[j] not in leetlist:
                leetlist.append(leet[j])

    return leetlist

# Strip pucnctuation.

def StripPunctuation(text):
    # Define punctuation and high ASCII characters
    punctuation=string.punctuation
    high_ascii_chars=''.join(chr(i) for i in range(128, 256))

    # Create a translation table to map all punctuation and high ASCII characters to spaces
    translation_table=str.maketrans({**dict.fromkeys(punctuation, ' '), **dict.fromkeys(high_ascii_chars, ' ')})

    # Replace punctuation and high ASCII characters with spaces in the text
    cleaned_text=text.translate(translation_table)

    return cleaned_text

# Json filter
# Filter end of line and hard spaces

def jsonFilter(s,FilterSpace=True):
    d=s.replace("\\n","").replace("\\t","").replace("\\r","")

    if FilterSpace==True:
        filterText='\t\r\n \u00A0'
    else:
        filterText='\t\r\n\u00A0'

    for c in filterText:
        d=d.replace(c,'')

    return(d)

# Break input into a word list for steering.

def GetWordList(text):
    words=text.lower().split()                          # Split the string by spaces
    return [word for word in words if word]     # Filter out empty strings (including spaces)

###
### Direct Companion functions
###

# Read the token file into a token dictionary.

def ReadTokens(gid=None):
    tokens={}
    if gid==None:
        tfile=RunningName+'.tokens'
    else:
        tfile=f"{ConfigStorage}/{gid}/{gid}.tokens"
    if os.path.exists(tfile):
        try:
            tokens=json.loads(jsonFilter(ReadFile(tfile)))
        except Exception as err:
            ErrorLog("Error token file is not in JSON format. Please see README.md for new layout")
            sys.exit(1)
    else:
        ErrorLog(f"Missing token file: {tfile}")
        sys.exit(1)

    if gid==None and 'Discord' not in tokens:
        ErrorLog("The MUST be a Discord API reference in the tokens file")
        sys.exit(1)

    return tokens

# Raw dump. For diagnostics purposes to see the actual return response from the AI model.

def RawLog(text):
    if LoggingLock.acquire(timeout=LoggingTimeout):
        try:
            mkdir(LoggingStorage)
            fn=f'{LoggingStorage}/RAWDUMP.log'
            fh=open(fn,'w')
            fh.write(text)
            fh.close()
        except:
            pass
        LoggingLock.release()

# Logging

def WriteLog(gid,uid,channel,text):
    if LoggingLock.acquire(timeout=LoggingTimeout):
        try:
            txt=text.replace('\n','\\n').replace('\r','\\r')

            time=(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'))

            s=f'{time} {uid} {channel} {txt}\n'

            dn=f'{LoggingStorage}/{gid}'
            mkdir(dn)
            fn=f'{dn}/{channel}.log'
            fh=open(fn,'a+')
            fh.write(s)
            fh.close()
        except Exception as err:
            print(f'LOG Broke: {err}')
            pass
        LoggingLock.release()

# Log errors

def ErrorLog(text):
    if LoggingLock.acquire(timeout=LoggingTimeout):
        try:
            txt=text.replace('\n','\\n').replace('\r','\\r')

            time=(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'))

            s=f'{time} {txt}\n'

            mkdir(LoggingStorage)
            fn=LoggingStorage+'/Errors.log'
            fh=open(fn,'a+')
            fh.write(s)
            fh.close()

            # print to console
            print(txt)
        except:
            pass
        LoggingLock.release()

# Needed for OpenAI as there are limits with the number of "tokens" that can be processed in a single
# request. HuggingFace=Ollama, Together.AI -> NOT OpenAI.

def MaintainTokenLimit(Tokens,orgmessages,max_tokens=128000,engine='openai',model="gpt-4o",encoding=None,HuggingFace=False):
    def count_tokens():
        # Calculate current tokens in the message list
        if engine=='openai':    #not HuggingFace:
            current_tokens=sum(len(enc.encode(message["content"])) for message in messages)
        elif engine=='cohere':
            current_tokens=sum(len(co.tokenize(text=message["content"],model=model,offline=False).tokens) for message in messages)
        else:
            current_tokens=0
            for message in messages:
#                current_tokens=len(enc(message["content"], add_special_tokens=False)["input_ids"])
                current_tokens=int(len(message["content"])/4)
        return current_tokens

    if engine=='openai': #not HuggingFace:
        if encoding!=None:
            enc=tiktoken.get_encoding(encoding)
        else:
            enc=tiktoken.get_encoding(tiktoken.model.MODEL_TO_ENCODING[model])
    elif engine=='cohere':
        co=cohere.ClientV2(api_key=Tokens['Cohere'])
#    else:
        # Gated, needs env HF_HUB_READ_TOKEN
#        enc=AutoTokenizer.from_pretrained(encoding)

    # Make a separate working copy of the original messages.
    messages=orgmessages.copy()

    try:
        current_tokens=count_tokens()

        old_tokens=current_tokens
        if max_tokens==0:
            return messages

        # While tokens exceed the limit, remove elements
        while current_tokens > max_tokens:
            for i in range(len(messages) - 1):
                if i < len(messages) - 1:
                    if messages[i]['role'].lower()=="user" and messages[i+1]['role'].lower()=="assistant":
                        # Remove the pair (two items)
                        messages.pop(i)
                        messages.pop(i)
                        break
                    elif messages[i]['role'].lower()=="user" and messages[i+1]['role'].lower()=="user":
                        # Remove only one item if two adjacent items are user/user
                        messages.pop(i)
                        break

            # Recalculate current tokens after removal
            current_tokens=count_tokens()
            # Check for the situation that we can't reduce the number of tokens.
            if old_tokens==current_tokens and current_tokens>max_tokens:
                return None
            old_tokens=current_tokens
    except Exception as err:
        ErrorLog(f"MaintainTokens: {sys.exc_info()[-1].tb_lineno}/{err}")
        return None

    return messages

# Get "steering" prompts, if there are any

# Steering files are a way of providing reinforcement to a pattern or paticular question/response designed
# to maintain the persona. Often times this may be used to enforce platform TOS/AUP.

# While we can dive into the nuances on unbiased AI material, we'll just skip to the finality: NOTHING IS
# UNBIASED. This will NEVER change simply because of how we learn, develop, and grow both as a family and
# societal structure. Bias is AUTOMATIC.

# The steering principles demonstrated here are much like we go through as children. The txt file will have
# the question repeated multiple times, with multiple acceptable responses. This practice is virtually
# identical to how we learn as children.

def GetSteering(bot,input_text):
    wordlist=GetWordList(StripPunctuation(input_text))
    if wordlist==None or wordlist==[]:
        return None

    SteerDir=f"{CompanionStorage}/{bot['BotName']}/Steering"

    try:
        for word in wordlist:
            if os.path.isdir(SteerDir+'/'+word):
                SteerDir+='/'+word
            elif os.path.isfile(SteerDir+'/'+word+'.txt'):
                prompts=ReadFile(SteerDir+'/'+word+'.txt').strip().split('\n')
                return prompts
    except Exception as err:
        ErrorLog(f"Steering: {sys.exc_info()[-1].tb_lineno}/{err}")

    # No txt file found. No prompts available
    return None

# Read channel list and set bot persona

def GetCompanionPersona(gid,channel,nsfw=False,Welcome=False):
    # Get the list of channels and the bot name that is allowed in a given channel
    cfg=f"{ConfigStorage}/{gid}/{gid}.cfg"
    if not os.path.exists(cfg):
        print(f'NO configuration: {cfg}')
        return None

    try:
        Config=json.loads(jsonFilter(ReadFile(cfg)))
    except Exception as err:
        ErrorLog(f"{cfg} damaged: {sys.exc_info()[-1].tb_lineno}/{err}")
        return None

    if 'Channels' not in Config:
        Config['Channels']={}

    bot=Config.copy()

    # Welcome should really be Default. If more then one bot it listed, comma separated, one one random,
    # and vwerify that it has a welcome file.

    if Welcome==True:
        bot['BotName']=Config['Welcome']
        bot['Channel']=None
    else:
        Channels=Config['Channels']
        bot['Channel']=channel
        if channel in Channels:
            bot['BotName']=Channels[channel]
            bot['ResponseAllowed']='Yes'
        else:
            bot['BotName']=Config['Default']
            bot['ResponseAllowed']='No'

    # Sort out loading a persona by channel and NSFW possibilities.

    # possibilities are that a persona can be:
    #   1. global (SFW)
    #   2. global, NSFW
    #   3. channel, (SFW)
    #   4. channel, NSFW

    # System Role
    # Test channel NSFW
    BotSystem=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.{channel}.system.nsfw"
    if nsfw and os.path.exists(BotSystem):
        bot['System']=BotSystem
    else:
        # Test channel SFW
        BotSystem=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.{channel}.system"
        if os.path.exists(BotSystem):
            bot['System']=BotSystem
        else:
            # Test global NSFW
            BotSystem=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.system.nsfw"
            if nsfw and os.path.exists(BotSystem):
                bot['System']=BotSystem
            else:
                # global SFW
                bot['System']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.system"

    # Persona file
    # Test channel NSFW
    BotPersona=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.{channel}.persona.nsfw"
    if nsfw and os.path.exists(BotPersona):
        bot['Persona']=BotPersona
    else:
        # Test channel SFW
        BotPersona=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.{channel}.persona"
        if os.path.exists(BotPersona):
            bot['Persona']=BotPersona
        else:
            # Test global NSFW
            BotPersona=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.persona.nsfw"
            if nsfw and os.path.exists(BotPersona):
                bot['Persona']=BotPersona
            else:
                # global SFW
                bot['Persona']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.persona"

    bot['Welcome']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.welcome"
    bot['Vulgarity']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.vulgarity"
    bot['ScamURLS']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.scamurls"
    bot['AutoFilter']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.autofilter"
    bot['AgeExploit']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.ageexploit"
    bot['TooMuchInformation']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.tmi"
    bot['Broken']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.broke"
    bot['URLBroken']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.urlbroke"
    bot['YTtags']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.yttags"
    bot['noYTtags']=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.noyttags"

    # Load bot config file. Channel CFg always has priority.
    settings={}
    try:
        bcfg=f"{ConfigStorage}/{gid}/{bot['BotName']}.{channel}.cfg"
        if os.path.exists(bcfg):
            settings=json.loads(jsonFilter(ReadFile(bcfg)))
        else:
            bcfg=f"{ConfigStorage}/{gid}/{bot['BotName']}.cfg"
            if os.path.exists(bcfg):
                settings=json.loads(jsonFilter(ReadFile(bcfg)))
    except Exception as err:
        ErrorLog(f"{bcfg} damaged: {sys.exc_info()[-1].tb_lineno}/{err}")

    # Merge the persona settings into the controlling dictionary
    if settings!={}:
        bot|=settings

    # Run through some sanity checks.
    if 'AutoLogging' not in bot:
        bot['AutoLogging']='yes'
    if 'AutoModeration' not in bot:
        bot['AutoModeration']='yes'
    if 'AllowBot' not in bot:
        bot['AllowBot']='no'
    if 'Engine' not in bot:
        bot['Engine']="openai"
    if 'Model' not in bot:
        bot['Model']="gpt-4o-mini"
    if 'FreqPenality' not in bot:
        bot['FreqPenality']=0.67
    else:
        if type(bot['FreqPenality'])!=float:
            bot['FreqPenality']=float(bot['FreqPenality'])
    if 'Temperature' not in bot:
        bot['Temperature']=0.37
    else:
        if type(bot['Temperature'])!=float:
            bot['Temperature']=float(bot['Temperature'])
    if 'AllowVulgarity' not in bot:
        bot['AllowVulgarity']='No'
    if 'DeveloperUID' not in bot:
        bot['DeveloperUID']=0
    else:
        if type(bot['DeveloperUID'])!=int:
            bot['DeveloperUID']=int(bot['DeveloperUID'])
    if 'MaxMemory' not in bot:
        bot['MaxMemory']=100
    else:
        if type(bot['MaxMemory'])!=int:
            bot['MaxMemory']=int(bot['MaxMemory'])

    # return the current bot
    return bot

# This function handle the actual responses.

def GetBabble(message,text):
    try:
        # The "uid" and "channel" are used to load the persona and store the memory to
        # disk. At this point, we don't have any global variables.

        # EXTEND to: GuildID/ChannelID:NSFW/MessageID/UID/input

        dataline=text.split('/')
        gid=message.guild.id
        uid=dataline[0]
        channel,cnsfw=dataline[1].split(':')
        nsfw=(cnsfw=='T')
        input_text=str(dataline[2:]).replace("'","\'").replace('"',"'")

        # Don't remember this request. does NOT bypass logging

        ForgetThisMessage=False
        if text.strip().startswith('%#>'):
            ForgetThisMessage=True
            input_text=input_text[3:]

        # Load the persona and memory files
        # NSFW not allowed in threads directly, so find parent channel
        # Question: Do we take info of NSFW state or force verify? Currebt is face value
        bot=None
        if isinstance(message.channel,discord.Thread):
            pchannel=str(message.channel.parent)
            bot=GetCompanionPersona(gid,pchannel,nsfw)   # message.channel.parent.nsfw
        else:
            bot=GetCompanionPersona(gid,channel,nsfw)

        persona=[]
        mList=[]

        # Read system role from system tag file
        # { "role": "system", "content": ""}

        if os.path.exists(bot['System']):
            buff=ReadFile(bot['System']).replace('\n','\\n').replace("'","\'").replace('"',"'").strip()
            sysList=[ '{'+ f'"role": "system", "content": "{buff}"' +'}' ]
        else:
            sysList=[]

        # Read the persona file
        if os.path.exists(bot['Persona']):
            pfList=ReadFile(bot['Persona']).strip().split('\n')
        else:
            pfList=[]
        pList=sysList+pfList

        # Load memory files, if one exists.
        dn=f"{MemoryStorage}/{gid}/{bot['BotName']}"
        mkdir(dn)
        fn=f"{dn}/{bot['BotName']}.{uid}.{channel}.memory"
        if os.path.exists(fn):
            mList=ReadFile(fn).strip().split('\n')
            pList+=mList

        # Build the complete persona list

        for s in pList:
            try:
                persona.append(json.loads(s))
            except:
                ErrorLog(f"Broke: {sys.exc_info()[-1].tb_lineno}/{s}")

        # Look for a "steering" file. Should be last to ensurew its not chopped of.
        sList=GetSteering(bot,input_text)
        if sList!=None:
            pList+=sList

        # Add user input

        memU={ "role": "user", "content": input_text }
        persona.append(memU)

        # Process API for the request

        el=list(bot['Engine'].split(','))       # Engine list
        ec=len(el)                              # Engine list length
        tl=list(bot['MaxTokens'].split(','))    # Max Tokens list
        mt=len(tl)                              # Max Tokens list length
        # Encoding alwayys takes priority over modem
        if 'Encoding' in bot:
            ml=list(bot['Encoding'].split(','))        # Model list
            mc=len(ml)                              # Model list length
        else:
            ml=list(bot['Model'].split(','))        # Model list
            mc=len(ml)                              # Model list length

        # The number of models MUST equal the number of engines. 1 model per engine

        if ec!=mc!=mt:
            ErrorLog(f"Broke GB ((models/Encoding)!=engines!=MaxTokens): {sys.exc_info()[-1].tb_lineno}/{err}")
            return PickRandomResponse(bot['Broken'])

        # Run through the engins/models until we have a response.

        response=None
        ecounter=0
        while response==None and ecounter<ec:
            cbot=copy.deepcopy(bot)             # Make a copy
            cbot['Engine']=el[ecounter]
            cbot['MaxTokens']=int(tl[ecounter])
            # Encoding or Model
            if 'Encoding' in cbot:
                cbot['Encoding']=ml[ecounter]
            else:
                cbot['Model']=ml[ecounter]

            response=GetAIResponse(gid,persona,cbot)
            if response==None:
                ecounter+=1
            else:
                break

        # check to see if we got a response

        if response==None or response=='':
            return PickRandomResponse(bot['Broken'])

        # Save "memory" to disk

        memA={ "role": "assistant", "content": response }

        mList.append(json.dumps(memU))
        mList.append(json.dumps(memA))

        # Keep memory at a limited amount. The *2 is because we are saving user and AI responses.

        if bot['MaxMemory']>0 and memA['content']!=None and not ForgetThisMessage:
            if len(mList)>(bot['MaxMemory']*2):
                mList=mList[2:]

            fh=open(fn,'w')
            for i in mList:
                fh.write(i+'\n')
            fh.close()

        # Return the AI response

        return response
    except Exception as err:
        ErrorLog(f"Broke GB: {sys.exc_info()[-1].tb_lineno}/{err}")
        return PickRandomResponse(bot['Broken'])

# Handles the persona as a synchronous call. I chose this approach for symplicity and adaptability.

async def HandleOneMessage(request):
    global request_list

    with concurrent.futures.ThreadPoolExecutor() as pool:
        try:
            input_text=request['input']
            guild=client.get_guild(request['gid'])
            channel=await guild.fetch_channel(request['cid'])
            message=await channel.fetch_message(request['mid'])

            uid=str(message.author.id)
            author=str(message.author.mention)
            channel=str(message.channel)

            # NSFW not allowed in threads, so find parent channel
            bot=None
            if isinstance(message.channel,discord.Thread):
                pchannel=message.channel.parent
                bot=GetCompanionPersona(guild.id,str(pchannel),pchannel.nsfw)
                nsfw=str(pchannel.nsfw)[0]
                # WTAF? Really? bots have to join a thread...
                if not message.channel.me:
                    await message.channel.join()
            else:
                bot=GetCompanionPersona(guild.id,str(message.channel),message.channel.nsfw)
                nsfw=str(message.channel.nsfw)[0]
            if bot==None:
                return

            # Convert boolean to str as a passing argument.
            print('U',guild.id,nsfw,uid,message.author,message.channel,len(message.content))

            # Handle any vulgarity
            if bot['AutoModeration'].lower()!='no' \
            and os.path.exists(bot['Vulgarity']) \
            and bot['AllowVulgarity'].lower()=='no' \
            and not nsfw=='T':
                if bool(pc.predict([ input_text ]))==True:
                    await ModeratorNotify(bot,message.guild,f"{message.author.name}/{message.author.id} chastized for vulgarity in {message.channel.name}")
                    await send_response(bot,message,PickRandomResponse(bot['Vulgarity']),delete=57)
                    try: # delete the offending message
                        await message.delete()
                    except:
                        pass
                    return
                else:
                    istr=f"{uid}/{channel}:{nsfw}/{input_text}"
                    response=await client.loop.run_in_executor(pool, GetBabble,message, istr)
            else:
                istr=f"{uid}/{channel}:{nsfw}/{input_text}"
                response=await client.loop.run_in_executor(pool, GetBabble,message, istr)

            print('B',guild.id,client.user.id,bot['BotName'],message.channel,len(response))

            # Check to see if we actually got a response
            if response!=None and response.lower().strip()!="none":
                # Handle webhooks for response
                await send_response(bot,message,response)
            else:
                # Communication with AI failed. Put message back in queue
                if ResponseLock.acquire(timeout=ResponseTimeout):
                    msg={}
                    msg['input']=input_text
                    msg['gid']=guild.id
                    msg['cid']=message.channel.id
                    msg['mid']=message.id
                    request_list.append(msg)
                    ResponseLock.release()
                else:
                    ErrorLog("Lock failed Reput Message")
        except Exception as err:
            ErrorLog(f"Broken HandleOneMessage: {sys.exc_info()[-1].tb_lineno}/{err}")

###
### AI handlers (Optimized)
###

# General AI Call Handler
def GetAIResponse(gid,persona,bot):
    print(gid,bot.get('Model'), bot.get('Encoding'))
    provider=bot.get('Engine').lower()
    try:
        # Default connection timeout
        tout=bot.get('Timeout', 60)
        mt=bot.get('MaxTokens', 16000 if provider=='OpenAI' else 4096)
        if provider=='ollama' and mt==0:
            mt=2048

        model=bot.get('Model')
        encoding=bot.get('Encoding')
        seed=bot.get('Seed', 0)

        # Provider-specific handling
        Tokens=ReadTokens(gid)

        # Adjust persona based on token limit
        WorkingPersona=MaintainTokenLimit(Tokens, persona, max_tokens=mt, engine=provider, model=model, encoding=encoding, HuggingFace=(provider != 'openai'))
        if WorkingPersona is None:
            return PickRandomResponse(bot['TooMuchInformation'])

        if provider=='openai':
            clientAI=openai.OpenAI(api_key=Tokens['OpenAI'])
            completion=clientAI.chat.completions.create(
                model=model,
                frequency_penalty=bot['FreqPenality'],
                temperature=bot['Temperature'],
                messages=WorkingPersona,
                timeout=tout
            )
            clientAI.close()

        elif provider=='togetherai':
            clientAI=together.Together(api_key=Tokens['TogetherAI'], timeout=tout)
            completion=clientAI.chat.completions.create(
                model=model,
                frequency_penalty=bot['FreqPenality'],
                temperature=bot['Temperature'],
                messages=WorkingPersona,
                stream=False
            )

        elif provider=='cohere':
            co=cohere.ClientV2(api_key=Tokens['Cohere'], timeout=tout)
            completion=co.chat(
                model=model,
                frequency_penalty=bot['FreqPenality'],
                temperature=bot['Temperature'],
                messages=WorkingPersona,
                safety_mode="NONE"
            )

        elif provider=='ollama':
            options={
                "temperature": bot['Temperature'],
                "frequency_penalty": bot['FreqPenality'],
                "seed": seed,
                "num_ctx": mt
            }
            completion=ollama.chat(
                keep_alive=tout,
                stream=False,
                model=model,
                options=options,
                messages=WorkingPersona
            )

        # Parse response
        try:
            if provider=='openai' or provider=='togwetherai':
                response=completion.choices[0].message.content.strip()
            elif provider=='cohere':
                response=completion.message.content[0].text.strip()
            elif provider=='ollama':
                response=completion['message']['content'].strip()
            else:
                response=None
        except:
            response=None

        RawLog(f"{provider}/{model}: {str(completion)}")
        return response

    except Exception as err:
        ErrorLog(f"{provider} {model}: {sys.exc_info()[-1].tb_lineno}/{err}")
        return None

###
### Special features
###

def AnagramSolver(fromletters):
    def load_dictionary(file_path):
        # Load dictionary words from a file and return them as a set.
        with open(file_path, 'r') as f:
            words=set(word.strip().lower() for word in f.readlines())
        return words

    def find_valid_words_by_length(letters, dictionary):
        # Find all valid words from combinations and permutations of the input letters grouped by length.
        valid_words={}
        # Generate combinations of letters from 3 to the length of the input
        for length in range(3, len(letters) + 1):
            for combo in itertools.combinations(letters, length):
                # For each combination, generate all permutations
                for perm in itertools.permutations(combo):
                    word=''.join(perm)
                    if word in dictionary:
                        if length not in valid_words:
                            valid_words[length]=set()
                        valid_words[length].add(word)
        # Sort each length's word list alphabetically
        for length in valid_words:
            valid_words[length]=sorted(valid_words[length])
        return valid_words

    # Load letters from the command line argument
    letters=fromletters.lower()

    # Load dictionary from the dictionary.txt file
    dictionary=load_dictionary(AnagramWordList)

    # Find valid words from the provided letters, grouped by length
    valid_words_by_length=find_valid_words_by_length(letters, dictionary)

    # Sorting the dictionary by word length in descending order and printing results
    wstr=''
    for length in sorted(valid_words_by_length.keys(), reverse=True):
        wstr+=f"**Words of length {length}**:\n"
        wstr+=' '.join(valid_words_by_length[length])+'\n\n'
    return wstr

# Read tags from a YouTube video

def yttags2text(url):
    def extract_video_id(url):
        # Match either a youtu.be or youtube.com URL to extract the video ID
        youtube_regex=r"(?:youtu\.be\/|youtube\.com\/(?:.*v=|.*\/|.*v\/|.*embed\/|.*shorts\/))([a-zA-Z0-9_-]{11})"
        match=re.search(youtube_regex, url)

        if match:
            return match.group(1)
        else:
            return None

    def get_video_tags(video_id):
        # Fetch video details including tags
        try:
            request=youtube.videos().list(
                part="snippet",
                id=video_id
            )
        except Exception as err:
            ErrorLog(f"Broke yttags: {sys.exc_info()[-1].tb_lineno}/{err}")
            return '{[(*VNF*)]}'

        response=request.execute()

        if "items" in response and len(response["items"]) > 0:
            video_snippet=response["items"][0]["snippet"]
            if "tags" in video_snippet:
                tags=video_snippet["tags"]
                return tags
            else:
                return None
        else:
            return '{[(*VNF*)]}'

    try:
        # Create YouTube Object
        youtube=build('youtube', 'v3', developerKey=Tokens['YouTube'])

        # Retrieve and display tags
        tags=get_video_tags(extract_video_id(url))

        if isinstance(tags, list):
            return '\n'.join(tags)
        return tags
    except Exception as err:
        ErrorLog(f"Broke yttags: {sys.exc_info()[-1].tb_lineno}/{err}")
        return '{[(*VNF*)]}'

# Read a YouTube URL and pull the transcript

def youtube2text(video_url):
    try:
        # Extract the video ID from the URL
        #video_id=video_url.split('v=')[1]
        video_id=re.search(r'(v=|be/|embed/|v/|youtu\.be/|\/videos\/|\/shorts\/|\/watch\?v=|\/watch\?si=|\/watch\?.*?&v=)([a-zA-Z0-9_-]{11})',video_url).group(2)

        # Fetch the transcript using the YouTubeTranscriptApi
        transcript_list=youtube_transcript_api.YouTubeTranscriptApi.get_transcript(video_id)

        # Use the TextFormatter to convert the transcript into plain text
        transcript_text='\n'.join(line['text'] for line in transcript_list)

        return 'Video Transcript: '+transcript_text
    except Exception as err:
        ErrorLog(f"Broke youtube2text: {sys.exc_info()[-1].tb_lineno}/{err}")
    return None

# Read PDF from buffer

def PDF2Text(pdf_buffer):
    with pdfplumber.open(io.BytesIO(pdf_buffer)) as pdf:
        text=""
        for page in pdf.pages:
            text+=page.extract_text()
        return 'PDF Content: '+text

# Get a URL and scrub it to pure text

def html2text(url):
    try:
        # Set the user agent
        headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
        request=urllib.request.Request(url, headers=headers)

        print("Fetch:",url)

        # Video transcript?
        if 'youtube.com/watch' in url or 'youtu.be/' in url:
            input_text=youtube2text(url)
            return input_text

        # Not a YouTube transcript, Fetch the HTML content from the URL

        try:
            with urllib.request.urlopen(request,timeout=60) as response:
                html=response.read()
        except Exception as err:
            print("URL Error:",url,str(err))
            return None

        if response.code>=400:
            print("URL Code:",response.code,url)
            return None

        # Check for PDF signature
        if html[:5]=='%PDF-':
            text=PDF2Text(html).strip()
            print("PDF:",len(text),url)
            return text

        # Decoding MUST be done AFTER pdf test
        html=html.decode('utf-8',errors='ignore')

        # Remove the entire head section
        html=re.sub(r'<head.*?>.*?</head>', '', html, flags=re.DOTALL)

        # Remove script and style elements
        html=re.sub(r'<(script|style).*?>.*?</\1>', '', html, flags=re.DOTALL)

        # Reduce <a> elements to their text content
        html=re.sub(r'<a[^>]*>(.*?)</a>', r'\1', html, flags=re.DOTALL)

        # Remove all other HTML tags
        text=re.sub(r'<[^>]+>', '', html)

        # Remove extra whitespace
        text=re.sub(r'\s+', ' ', text).strip()

        print("URL:",len(text),url)

        return 'Web Page Content: '+text
    except Exception as err:
        ErrorLog(f"Broke html2text: {sys.exc_info()[-1].tb_lineno}/{err}")

    return None

###
### Discord support functions
###

# Disect discord messages. Forensic logging

def DisectMessage(event,message):
    if DisectLock.acquire(timeout=DisectTimeout):
        # Set up convience variables
        gid=message.guild.id
        channel=str(message.channel)
        uid=str(message.author.id)
        author=message.author
        member=message.guild.get_member(author.id) if message.guild else None
        nickname=member.nick if member and member.nick else author.name

        try:
            # Checking different channel types with isinstance
            if isinstance(message.channel, discord.TextChannel):
                insttype="Text Channel"
            elif isinstance(message.channel, discord.VoiceChannel):
                insttype="Voice Channel"
            elif isinstance(message.channel, discord.Thread):
                insttype="Thread"
            elif isinstance(message.channel, discord.DMChannel):
                insttype="Direct Message"
            elif isinstance(message.channel, discord.CategoryChannel):
                insttype="Category Channel"
            else:
                insttype="Unknown Channel"

            WriteLog(gid,uid,channel,f"Event trigger: {event}")
            WriteLog(gid,uid,channel,f"Message: {message.id}/{message.type}/{insttype}")
            WriteLog(gid,uid,channel,f"Message Timestamp: {message.created_at}/{message.edited_at if message.edited_at else 'Not Edited'}")
            WriteLog(gid,uid,channel,f"Message Pinned: {message.pinned}")
            WriteLog(gid,uid,channel,f"Message Author: {message.author.name}#{message.author.discriminator}/{nickname}/{author.display_name}/{message.author.id} Bot: {message.author.bot}")
            WriteLog(gid,uid,channel,f"Message Channel: {message.channel.name}/{message.channel.id}")
            WriteLog(gid,uid,channel,f"Message Guild: {message.guild.name if message.guild else 'DM'}/{message.guild.id if message.guild else 'DM'}")
            # Check if the message was sent via a webhook
            if message.webhook_id:
                WriteLog(gid,uid,channel,f"Message sent via Webhook ID: {message.webhook_id}")
            # List the users mentioned, if any
            if message.mentions:
                WriteLog(gid,uid,channel,f"Message Mentions: {[str(user) for user in message.mentions]}")
            if message.role_mentions:
                WriteLog(gid,uid,channel,f"Message Mentioned Roles: {[role.name for role in message.role_mentions]}")
            if message.channel_mentions:
                WriteLog(gid,uid,channel,f"Message Mentioned Channels: {[channel.name for channel in message.channel_mentions]}")
            if str(message.content).strip()!='':
                WriteLog(gid,uid,channel,f"Message Content: {message.content}")
            else:
                WriteLog(gid,uid,channel,f"Message Content: Empty content field")
            if message.reactions:
                WriteLog(gid,uid,channel,f"Message Reactions: {[(reaction.emoji, reaction.count) for reaction in message.reactions]}")

            # Log sticker information
            if message.stickers:
                WriteLog(gid,uid, channel, f"Message Stickers: {[{'name': sticker.name, 'id': sticker.id} for sticker in message.stickers]}")

            # Log reference information (for replies)
            if message.reference:
                ref=message.reference
                WriteLog(gid,uid, channel, f"Message is a Reply to: {ref.message_id} in Channel: {ref.channel_id} of Guild: {ref.guild_id}")

            # Log details about each embed
            if message.embeds:
                WriteLog(gid,uid,channel,f"Message Embeds: {len(message.embeds)} embeds")
                for index, embed in enumerate(message.embeds):
                    WriteLog(gid,uid, channel, f"Embed {index + 1} Title: {embed.title}")
                    WriteLog(gid,uid, channel, f"Embed {index + 1} Type: {embed.type}")
                    WriteLog(gid,uid, channel, f"Embed {index + 1} Description: {embed.description}")
                    WriteLog(gid,uid, channel, f"Embed {index + 1} URL: {embed.url}")
                    WriteLog(gid,uid, channel, f"Embed {index + 1} Timestamp: {embed.timestamp}")
                    WriteLog(gid,uid, channel, f"Embed {index + 1} Color: {embed.color}")
                    WriteLog(gid,uid, channel, f"Embed {index + 1} Footer: {embed.footer.text if embed.footer else 'None'}")
                    WriteLog(gid,uid, channel, f"Embed {index + 1} Image: {embed.image.url if embed.image else 'None'}")
                    WriteLog(gid,uid, channel, f"Embed {index + 1} Thumbnail: {embed.thumbnail.url if embed.thumbnail else 'None'}")
                    WriteLog(gid,uid, channel, f"Embed {index + 1} Author: {embed.author.name if embed.author else 'None'}")
                    WriteLog(gid,uid, channel, f"Embed {index + 1} Fields: {[{'name': field.name, 'value': field.value} for field in embed.fields]}")

            # Log detailed information about each attachment
            if message.attachments:
                WriteLog(gid,uid,channel,f"Message Attachments: {[attachment.url for attachment in message.attachments]}")
                for index, attachment in enumerate(message.attachments):
                    WriteLog(gid,uid, channel, f"Attachment {index + 1} Filename: {attachment.filename}")
                    WriteLog(gid,uid, channel, f"Attachment {index + 1} Size: {attachment.size} bytes")
                    WriteLog(gid,uid, channel, f"Attachment {index + 1} URL: {attachment.url}")
                    WriteLog(gid,uid, channel, f"Attachment {index + 1} Proxy URL: {attachment.proxy_url}")
                    WriteLog(gid,uid, channel, f"Attachment {index + 1} Height: {attachment.height if attachment.height else 'N/A'}")
                    WriteLog(gid,uid, channel, f"Attachment {index + 1} Width: {attachment.width if attachment.width else 'N/A'}")

            # Log message components (if any)
            if message.components:
                WriteLog(gid,uid, channel, f"Message Components: {[{'type': component.type, 'custom_id': component.custom_id} for component in message.components]}")
            WriteLog(gid,uid,channel,f"{'-'*80}")
        except AttributeError as e:
            WriteLog(gid,uid,channel,f"Error accessing message attributes: {e}")
        DisectLock.release()

# Change the nick name and avatar

async def ChangeNickAvatar(bot,message):
    # Only change avatar if channels change.
    channel=str(message.channel)
    cname=ReadFile(RunningName+'.lastchannel')
    if cname==None or channel.strip()!=cname:
        WriteFile(RunningName+'.lastchannel',channel+'\n')

        # Only seems to work with Administrator priviledges
        # Change the nickname if the bot.
        try:
            # Add code to check avatar name and channel. Only change avatar when
            # channel changes.
            await message.guild.me.edit(nick=bot['BotName'])
            pname=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.png"
            png=ReadFile(pname,binary=True)
            await client.user.edit(avatar=png)
        except Exception as err:
            ErrorLog(f"Broken Nick/Avatar: {sys.exc_info()[-1].tb_lineno}/{err}")

# Return the response to the user. messages over 1997 characters are returned as a
# message file attachment. Adapt to webhooks...

# If message is None, use system channel

async def send_response(bot,message,response,embed=None,delete=None,member=None):
    global delete_list
    whmsg=[]

    try:
        if bot['Channel']==None:
            webhooks=await member.guild.system_channel.webhooks()
        else:
            # If its a thread, get the parent channel
            pchannel=message.channel
            if isinstance(message.channel,discord.Thread):
                pchannel=message.channel.parent
            webhooks=await pchannel.webhooks()

        # Delete all webhooks with the name "Companion Temporary Webhook"
        for webhook in webhooks:
            if webhook.name=='Companion Temporary Webhook':
                await webhook.delete()

        threadID=None
        # Create a new webhook
        if bot['Channel']==None:
            syschannel=member.guild.system_channel
            webhook=await syschannel.create_webhook(name='Companion Temporary Webhook')
            author=None
        else:
            # Handle thread issues, get parent channel
            pchannel=message.channel
            if isinstance(message.channel,discord.Thread):
                pchannel=message.channel.parent
            webhook=await pchannel.create_webhook(name='Companion Temporary Webhook')
            # Get the name of the person this message is going to
            author=str(message.author.mention)
            if isinstance(message.channel,discord.Thread):
                threadID=message.channel # Actual thread ID

        # If the message is less then 1900 characters, just send it with a reply.
        if embed!=None:
            if author!=None:
                x1=f"{author}"
            else:
                x1=''
            if threadID:
                wm=await webhook.send(content=x1,embed=embed,username=bot['BotName'],avatar_url=bot['Avatar'],thread=threadID,wait=True)
            else:
                wm=await webhook.send(content=x1,embed=embed,username=bot['BotName'],avatar_url=bot['Avatar'],wait=True)
            whmsg.append(wm)
        elif len(response)<=1900:
            if author!=None:
                x1=f"{author} {response}"
            else:
                x1=f"{response}"
            # sending with and without threads must be separate
            if threadID:
                wm=await webhook.send(content=x1,username=bot['BotName'],avatar_url=bot['Avatar'],thread=threadID,wait=True)
            else:
                wm=await webhook.send(content=x1,username=bot['BotName'],avatar_url=bot['Avatar'],wait=True)
            whmsg.append(wm)
        else:
            # Break this up into multiple messages. Discord does NOT allow direct reply with multiple
            # message parts.
            x1=f"{author} {response}"
            l=len(x1)
            while l>1900:
                # We need to deal with the possibility that a \n doesn't exist in the input data.
                pmax=1900
                # Look for new line first
                p=x1.rfind('\n',0,pmax)
                if p==-1:
                    # Look for a period  Separate at a sentence
                    p=x1.rfind('.',0,pmax)
                    if p==-1:
                        # Look for a space  Separate at a word
                        p=x1.rfind(' ',0,pmax)
                        if p==-1:
                            # Brute split
                            pmax=1900
                if len(x1[:p])>0:
                    if threadID:
                        wm=await webhook.send(content=x1[:p].strip(),username=bot['BotName'],avatar_url=bot['Avatar'],thread=threadID,wait=True)
                    else:
                        wm=await webhook.send(content=x1[:p].strip(),username=bot['BotName'],avatar_url=bot['Avatar'],wait=True)
                    whmsg.append(wm)
                if x1[p]=='.' or x1[p]==' ':
                    p+=1
                x1=x1[p:].strip()
                l=len(x1)
            if l>0:
                if threadID:
                    wm=await webhook.send(content=x1.strip(),username=bot['BotName'],avatar_url=bot['Avatar'],thread=threadID,wait=True)
                else:
                    wm=await webhook.send(content=x1.strip(),username=bot['BotName'],avatar_url=bot['Avatar'],wait=True)
                whmsg.append(wm)

        # Queue autodelete messages. Responses can be multiple messages.
        if delete!=None:
            if DeleteLock.acquire(timeout=DeleteTimeout):
                for whm in whmsg:
                    delete_list.append({"gid": whm.guild.id,"cid":whm.channel.id,"mid":whm.id,"Expires":whm.created_at.timestamp()+delete } )
                DeleteLock.release()
            else:
                ErrorLog("Lock failed SR autodelete")
        # Delete the webhook
        await webhook.delete()
    except discord.errors.HTTPException as err:
        ErrorLog(f'Error sending message: {sys.exc_info()[-1].tb_lineno}/{err}')
    except Exception as err:
        ErrorLog(f'Error sending message: {sys.exc_info()[-1].tb_lineno}/{err}')

# Post message is moderation area

async def ModeratorNotify(bot,guild,text):
    print(text)
    # If there is a moderator area, send a message to it.
    if 'ModerationArea' in bot:
        for channel in guild.channels:
            if isinstance(channel,discord.TextChannel) and channel.name==bot['ModerationArea']:
                await channel.send(text)
                break

###
### Background Tasks
###

# This task sweeps all memry fils in a given server and purges any memory file over
# X days. Housekeeping is required otherwise users will burn storage and there MUST
# be reasonable limits. Rune ONCE a day.

@tasks.loop(hours=23)
async def MemoryMaintenance():
    # Get server CFg files
    cfg_files=[os.path.join(ConfigStorage, file) for file in os.listdir(ConfigStorage) if file.endswith('.cfg')]

    # Read server CFG file and get MaxMemory
    for cfg in cfg_files:
        try:
            Config=json.loads(jsonFilter(ReadFile(cfg)))
            await asyncio.sleep(0)
        except Exception as err:
            ErrorLog(f"{cfg} damaged: {sys.exc_info()[-1].tb_lineno}/{err}")
            continue

        # Get how long to keep old memory files in seconds
        MaxMemory=float(Config['MaxMemory'])*86400
        sid=cfg.split('/')[5].split('.')[0]
        dn=f"{MemoryStorage}/{sid}"

        # Get list of .memory files for this server

        mfiles=[]
        for dirpath, _, filenames in os.walk(dn):
            for file in filenames:
                if file.endswith('.memory'):
                    mfiles.append(os.path.join(dirpath, file))

        # Now check the memory files for age. If the file is OLDER then MaxMemory,
        # delete it.

        ct=time.time()
        for mf in mfiles:
            lmt=os.path.getmtime(mf)
            if ct-lmt>MaxMemory:
                os.remove(mf)
            await asyncio.sleep(0)

# Handle autodelete messages. It is important to consider that when this is executed,
# the original message may not exist. We have to fetch message from the guild,
# channel, and message IDs.

@tasks.loop(seconds=3)
async def autodelete_messages():
    global delete_list

    if DeleteLock.acquire(timeout=DeleteTimeout):
        try:
            now=time.time()
            if delete_list:
                for msg in list(delete_list):
                    if now>msg['Expires']:
                        guild=client.get_guild(msg['gid'])              # Guild
                        channel=guild.get_channel(msg['cid'])           # Channel
                        message=await channel.fetch_message(msg['mid']) # Message

                        delete_list.remove(msg)
                        await message.delete()
        except Exception as err:
            ErrorLog(f'Broke ADM: {sys.exc_info()[-1].tb_lineno}/{err}')
        DeleteLock.release()
    else:
        ErrorLog("Lock failed ADM")

# The dirty part. I wanted to serialize requests for the bot, so i timer approach is used coupled with
# a locking method. This keep a rate limit approach for the open AI api in accordance to its policies.
# Sleeping method can also be added.

@tasks.loop(seconds=5)
async def update_response_data():
    global request_list

    # Pull the next request

    request=None
    if ResponseLock.acquire(timeout=ResponseTimeout):
        if request_list:
            request=request_list.pop(0)

        ResponseLock.release()

    # Only 1 AI request at a time

    if request!=None:
        if BabbleLock.acquire(timeout=BabbleTimeout):
            try:
                await HandleOneMessage(request)
            except Exception as err:
                ErrorLog(f"Broke: {sys.exc_info()[-1].tb_lineno}/{str(err)}")
            BabbleLock.release()
        else:
            ErrorLog("Lock Failed URD")

# This function cleans up the active user lists.

# The reduce factor weighs in on the user typing within the channel. It
# acts as a cool down to prevent a channel raid. and keep the slowmode
# appripriate to the channel "pressure".

@tasks.loop(seconds=60)
async def track_active_users():
    global active_users

    try:
        current_time=time.time()
        # Sweep every channel
        for ac in list(active_users):
            # Sweep users in a channel
            for au in list(active_users[ac]):
                # Check time added againt expiration
                if (current_time-au[1])>SLOW_MODE_COOLDOWN:
                    # reduce factor
                    au[2]-=1
                    if au[2]<1:
                        active_users[ac].remove(au)
                    else:
                        au[1]=time.time()
    except Exception as err:
        ErrorLog(f'TAU Broke: {sys.exc_info()[-1].tb_lineno}/{err}')

###
### Discord event functions
###

# This section deals with a particularly visicous attack where the attacker asks an innocent question like "how
# many sides to an octogon?"

# The victim will respond 8.

# The attacker ffthen edits the question to read "How old are you?" or "what is your age?" and takes a screen shot.
# The attacker then uses the screen shot to get the victim permenantly banned, as it looks like the victim
# violated Discord's age restrictions. The is a zero tolorence policy for Discord and a viscious way to target
# people.

# There is one serious restriction to this. Is a message is posted prior to the bot being loaded, but edited
# after the fact, the bot won't receive the edit notification. THIS IS A MAJOR LOOPHOLE, but one only DISCORD can
# fix.

# It seems one of the best ways to defeat this is just to delete the original message.

@client.event
async def on_message_edit(before, after):
    # Search for word in list
    def CheckWordList(side,sampleText):
        Found=False
        for w in range(len(side)):
            if side[w] in sampleText:
                Found=True
                break
        return Found

    try:
        if after.content==before.content:
            return

        # If this is a thread, get the parent channel
        pchannel=after.channel
        nsfw=None
        if isinstance(after.channel,discord.Thread):
            pchannel=after.channel.parent
            nsfw=after.channel.parent.nsfw
        else:
            nsfw=after.channel.nsfw

        channel=str(pchannel)
        uid=str(after.author.id)
        bot=GetCompanionPersona(after.guild.id,channel,nsfw)
        if bot==None:
            return

        # Log everything
        if bot['AutoLogging'].lower()!='no':
            DisectMessage("Message edited (before)",before)
            DisectMessage("Message edited (after)",after)

        # Manage automated moderation
        if bot['AutoModeration'].lower()=='no':
            return

        # Build leet lists

        leftside=BuildLeetList(lside)
        rightside=BuildLeetList(rside)

        sampleText=StripPunctuation(after.content.lower())

        leftFound=CheckWordList(leftside,sampleText)
        rightFound=CheckWordList(rightside,sampleText)

        # NEED a global moderation switch
        # Check for vulgarity in the edited response

        # This could be a problem at the global level of a server, where partial moderation may be desired,
        # vs an absolute approach.

        if bot['AutoModeration'].lower()!='no' \
        and os.path.exists(bot['Vulgarity']) \
        and bot['AllowVulgarity'].lower()!='yes' \
        and not nsfw:
            if bool(pc.predict([ after.content ]))==True:
                await ModeratorNotify(bot,after.guild,f"{after.author.name}/{after.author.id} chastized for vulgarity in {after.channel.name}")
                await send_response(bot,after,PickRandomResponse(bot['Vulgarity']),delete=57)
                try: # delete the offending message
                    await after.delete()
                except:
                    pass
                return

        if leftFound and rightFound:
            await ModeratorNotify(bot,after.guild,f"LEET words/Malicious edit possibility: {after.author.name}/{after.author.id} in {after.channel.name}")
            # This seems to be a better way to ruin the attacker's plan
            embed=discord.Embed(
                title='Message edited',
                description=f'UID: {uid}\nAuthor: {after.author}\n\nBefore edit:\n\n{before.content}\n\nAfter edit:\n\n{after.content}\n\n',
                color=discord.Color.red() )
#            try: # delete the edited message
#                await after.delete()
#            except:
#                pass
            await send_response(bot,after,None,embed=embed)
            return

    except Exception as err:
        ErrorLog(f'Broken Edit: {sys.exc_info()[-1].tb_lineno}/{err}')

# Log deleted messages. Nothing escapes the watchful eye of the moderation system

@client.event
async def on_message_delete(message):
    # Figure out which persona is calling the shots.
    channel=str(message.channel)
    uid=str(message.author.id)
    bot=GetCompanionPersona(message.guild.id,channel,message,channel.nsfw)
    if bot==None:
        return

    # Log everything
    if bot['AutoLogging'].lower()!='no':
        DisectMessage("Deleted message",message)

# Make a nice announcement welcoming the user to the server.

# Currently, the last avtive personality will be the welcoming party. Really should force load a bot
# personality by name.

@client.event
async def on_member_join(member):
    try:
        # Welcome forces the default bot.
        bot=GetCompanionPersona(member.guild.id,None,Welcome=True)
        if bot==None:
            return

        if bot['AutoLogging'].lower()!='no':
            ErrorLog(f"OMJ: {member}")

        # If there is a moderator area, send a message to it.
        await ModeratorNotify(bot,member.guild,f"{member.mention}/{member.id} joined.")

        if os.path.exists(bot['Welcome']):
            # Figure out which persona is calling the shots.
            response=PickRandomResponse(bot['Welcome']).replace('{username}',f'**{member.mention}**')
            # A message of None will automatically refer to the system channel
            await send_response(bot,None,response,member=member)
    except Exception as err:
        ErrorLog(f'OMJ: Error sending message: {sys.exc_info()[-1].tb_lineno}/{err}')

# Record the user leaving the server in the designated moderator area.

@client.event
async def on_member_remove(member):
    try:
        # Welcome forces the default bot.
        bot=GetCompanionPersona(member.guild.id,None,Welcome=True)
        if bot==None:
            return

        if bot['AutoLogging'].lower()!='no':
            ErrorLog(f"OMR: {member}")

        # If there is a moderator area, send a message to it.
        await ModeratorNotify(bot,member.guild,f"{member.mention}/{member.id} left.")
    except Exception as err:
        ErrorLog(f'OMR: Error sending message: {sys.exc_info()[-1].tb_lineno}/{err}')

# Implement an auto slowmode based on actual usera typing per channel
# NOT bot dependant... RETHINK this for CFG settings and enable/disable.

@client.event
async def on_typing(channel, user, when):
    global active_users

    try:
        # if direct message, ignore
        if isinstance(channel,discord.DMChannel):
            return

        # If this is a thread, get the parent channel
        pchannel=channel
        if isinstance(channel,discord.Thread):
            pchannel=channel.parent

        # Create a unique key for the channel using guild and channel IDs
        key = f"{pchannel.guild.id}.{pchannel.id}"

        # When this function gets called, someone was typing. Add them to the channel
        # tracking list.

        if key not in active_users:
            active_users[key]=[]

        found=False
        for u in active_users[key]:
            if u[0]==user.id:
                # Reset the timer
                u[1]=time.time()
                u[2]+=1
                found=True

        if not found:
            # User ID, and the time they were typing, factor
            # The factor is used as a decrementer to prevent manipulation
            ut=[user.id,time.time(),1]
            active_users[key].append(ut)

        # Calculate the number of typing users in the channel
        typing_count=len(active_users[key])

        # Check if the cooldown has expired. Slowdown can only be adjust ONCE every 5
        # minutes per channel.

        current_time = time.time()
        last_change = last_slow_mode_change.get(key, 0)
        if current_time - last_change < SLOW_MODE_COOLDOWN:
            return  # Skip the adjustment process if still in cooldown

        # Calculate expected users
        server = channel.guild

        # Theory: 1% of all active users throughout the server should be active
        # normally. Assuming the distribution of users will be spread out over the
        # number of channels, gives us an expected number of users per channel. This
        # is a fallacy, but it is a reasonable starting point to build the theory on.
        # Possible expansions could be actually messuring user activity per channel,
        # but to get something very basic, I felt this was reasonable enough for most
        # applications given that I am using only 3 secords per user as the slowdown
        # rate, unless the number of user is above the expected amount. This is a
        # primitive anti-raid method.

        expected_users = int(max(1, round(len(server.members) * 0.01 / len(server.channels))))
        if expected_users<1:
            expected_users=1

        # Determine slow mode duration
        if typing_count > 2 * expected_users:  # Over threshold
            slow_mode_duration = SLOW_MODE_DURATION * typing_count * 10
        elif typing_count > expected_users:
            slow_mode_duration = SLOW_MODE_DURATION * typing_count
        else:
            slow_mode_duration = 0

        # Apply slow mode
        await pchannel.edit(slowmode=slow_mode_duration)
        last_slow_mode_change[key] = current_time  # Update the last change time
        print(f"Slowmode {server.id}/{channel.name}: {slow_mode_duration} seconds, {typing_count} active, {expected_users} expected")
    except Exception as err:
        ErrorLog(f'OnType Broke: {sys.exc_info()[-1].tb_lineno}/{err}')

# Get the user message and add it to a processing list. This really is the driving point of the AI
# responses. Currently it responds to any message in the designated areas, but it could easily be
# adapted to respond to message of a particular interest, like cooking, or programming.

@client.event
async def on_message(message):
    global request_list

    try:
        # Ignore messages from the bot itself
        if message.author==client.user:
            return

        # Handle direct messages
        if isinstance(message.channel,discord.DMChannel):
            await message.channel.send(f'You have messaged the Companion AI chatbot/moderation. This bot functions ONLY within the limits of a Discord server and does NOT support interactions via DMs. Thank you.')
            return

        # Figure out which persona is calling the shots.
        channel=str(message.channel)
        uid=str(message.author.id)

        # Handle auto publish
        if message.channel.type==discord.ChannelType.news:
            await message.publish()

        # NSFW not allowed in threads, so find parent channel
        bot=None
        nsfw=False
        if isinstance(message.channel,discord.Thread):
            pchannel=str(message.channel.parent)
            nsfw=message.channel.parent.nsfw
            bot=GetCompanionPersona(message.guild.id,pchannel,nsfw)
            # WTAF? Really? bots have to join a thread...
            if not message.channel.me:
                await message.channel.join()
        else:
            nsfw=message.channel.nsfw
            bot=GetCompanionPersona(message.guild.id,channel,nsfw)
        if bot==None:
            return

        # Log everything
        if bot['AutoLogging'].lower()!='no':
            DisectMessage("Received message",message)

        # MUST BE FIRST!
        # Imposter among us no more!
        # Check to see if the user trying to impersonate the owner
        guild=message.guild
        owner=guild.owner
        owner_name=owner.name.lower()
        owner_nick=owner.display_name.lower()
        member_name=message.author.name.lower()
        member_nick=message.author.display_name.lower()

        # Check for exact matches (username or display name).
        if message.author!=owner and (member_name==owner_name or member_nick==owner_nick):
            # If the names are identical, check the avatar.
            # Fetch URLs for avatars
            member_avatar_url=message.author.display_avatar.url
            owner_avatar_url=owner.display_avatar.url

            # Load images for comparison
            member_avatar_image=LoadImageURL(member_avatar_url)
            owner_avatar_image=LoadImageURL(owner_avatar_url)

            # Convert avatars to perceptual hashes
            member_hash=imagehash.phash(member_avatar_image)
            owner_hash=imagehash.phash(owner_avatar_image)

            # Compare hash distance (simple comparison)
            if member_hash==owner_hash:
                # Send embed indicating imposter before deleting the message
                embed=discord.Embed(
                    title="🚨 IMPOSTER DETECTED 🚨",
                    description=f"User {message.author.mention} (**ID: {message.author.id}**) is impersonating the server owner.\n\n"
                                f"**THIS IS NOT THE OWNER**. DO NOT ACCEPT DIRECT MESSAGES FROM THIS USER!",
                    color=discord.Color.red()
                )
                embed.add_field(name="Imposter Message", value=message.content, inline=False)
                embed.set_footer(text="Action: Message will be deleted, and user will be banned.")

                # Send the embed to the channel where the message was sent
                await send_response(bot,message,None,embed=embed,delete=300)
                # Delete the imposter's message
                await message.delete()
                # Kick the imposter
                await message.author.ban(reason="Impersonating the server owner")
                return

        # Check for input that is a number only
        if bot['AutoModeration'].lower()!='no' and os.path.exists(bot['AgeExploit'])==True and NumberOnly(message.content):
            await ModeratorNotify(bot,message.guild,f"AgeExploit detected: {message.author.name}/{message.author.id} in {message.channel.name}")
            author=str(message.author.mention)
            # If its a bot, it doesn't deserve a response
            if not message.author.bot:
                await send_response(bot,message,PickRandomResponse(bot['AgeExploit']),delete=300)
            await message.delete()
            return

        # This is really the only absolute exception to BOT responsing...
        # Check for scam links and delete them.

        # BUILD FALSE POSITIVES LIST, so far I haven't ran into any

        if os.path.exists(CompanionScamURLS)==True and os.path.exists(bot['ScamURLS'])==True and '://' in message.content:
            scamurls=ReadFile2List(CompanionScamURLS)
            for url in scamurls:
                if url in message.content:
                    print(f"ScamURL: {url} {message.author.mention}")
                    author=str(message.author.mention)
                    # If its a bot, it doesn't deserve a response
                    if not message.author.bot:
                        await send_response(bot,message,PickRandomResponse(bot['ScamURLS']),delete=57)
                    await message.delete()
                    return

        if os.path.exists(CompanionAutoFilter)==True and os.path.exists(bot['AutoFilter'])==True:
            autofilter=ReadFile2List(CompanionAutoFilter)
            for text in autofilter:
                if text in message.content:
                    author=str(message.author.mention)
                    # If its a bot, it doesn't deserve a response
                    if not message.author.bot:
                        await send_response(bot,message,PickRandomResponse(bot['AutoFilter']),delete=57)
                    await message.delete()
                    return

        # Don't respond to other bots.
        if message.author.bot and bot['AllowBot'].lower()!='yes':
            return

        # Allow the user to erase their own stored conversation

        if str(message.content).strip().startswith('%Forget'):
            await message.delete()
            dn=f"{MemoryStorage}/{guild.id}/{bot['BotName']}"
            mkdir(dn)
            fn=f"{dn}/{bot['BotName']}.{uid}.{channel}.memory"
            if os.path.exists(fn):
                os.remove(fn)
                await send_response(bot,message,"Conversation forgotten",delete=57)
            return

        if 'DeveloperUID' in bot and int(uid)==bot['DeveloperUID']:
            if str(message.content).strip().startswith('%CheckBot'):
                member=message.guild.get_member(client.user.id) # Get the bot's member object in the guild
                # Check if the bot is a member of the channel
                if member in message.channel.members and bot['ResponseAllowed'].lower()=='yes':
                    await send_response(bot,message,"Allowed",delete=57)
                else:
                    await send_response(bot,message,"Not Allowed",delete=57)
                if request_list!=[]:
                    msg=f"{len(request_list)} pending requests"
                    await send_response(bot,message,msg,delete=57)
                await message.delete()
                return

            if str(message.content).strip().startswith('%PurgeRequests'):
                # Give the AI the message
                if ResponseLock.acquire(timeout=ResponseTimeout):
                    if request_list!=[]:
                        l=len(request_list)
                        request_list=[]
                        msg=f"{l} requests purged"
                    else:
                        msg=f"Request list already empty"
                    ResponseLock.release()
                    await send_response(bot,message,msg,delete=57)
                else:
                    ErrorLog("Lock failed OM")
                return

        # If we have a bot, then we can converse here.

        # Check to see if the message is a replay to the bot, or a mention or a
        # public message, specifically that the user used the reply feature of the
        # APP, or the @ and bot name, if talking to the bot. This convoluted
        # approach is needed because all bot replies are via webhooks. Webhooks
        # have different user IDs then the bot.

        # All of this is NECCESSARY if we want the ability to server multiple
        # personas, each with its own avatar. Discord does NOT allow multiple
        # avatars per a single user (bot.user) without changing EVERY avatar in
        # EVERY channel. Using webhooks is the only viable option to acheive
        # multiple AI personas in an imersive methodology.

        botReference=False
        if message.reference is not None and message.reference.message_id is not None:
            refmsg=await message.channel.fetch_message(message.reference.message_id)
            # ID won't work here because everything is sent via webhook
            if refmsg.author.name==client.user.name or refmsg.author.name==bot['BotName']:
                botReference=True

        publicMsg=message.reference==None and len(message.mentions)==0 and len(message.role_mentions)==0 and len(message.channel_mentions)==0
        botAnswer=publicMsg or botReference or client.user in message.mentions

        if bot['ResponseAllowed'].lower()=='yes' and bot['Channel']!=None and botAnswer:
            input_text=message.content.strip()

            # If a trigger file exists, check to see if one on the trigger words are in the user message.
            # Also if trigger file exists, and message is directed to bot.
            trigger=f"{CompanionStorage}/{bot['BotName']}/{bot['BotName']}.{channel}.trigger"
            if os.path.exists(trigger):
                # If message is directed to bot, override trigger
                if client.user in message.mentions or (message.reference and message.reference.resolved.author.id==client.user.id):
                    pass
                else:
                    wfound=False
                    twords=ReadFile2List(trigger,ForceLower=True)
                    inwords=StripPunctuation(input_text.replace('  ',' ')).split(' ')
                    while '' in inwords:
                        inwords.remove('')
                    # Search the list word by word
                    for w in inwords:
                        if w.lower() in twords:
                            wfound=True
                            break
                    # A trigger word was not found, don't continue.
                    if wfound==False:
                        return

            # Anagram solver... how the hell did I end up don the rabbit hole? Really?

            if os.path.exists(AnagramWordList) and input_text.lower().strip().startswith('%anagramsolver'):
                letters=input_text[14:].strip().lower()
                input_text=AnagramSolver(letters)
                await send_response(bot,message,input_text)
                return

            # Read tags from YouTune video
            if os.path.exists(bot['YTtags']) and os.path.exists(bot['noYTtags']) \
            and input_text.lower().strip().startswith('%yttags'):
                # Web/URL reference
                url=input_text[7:].strip()
                input_text=yttags2text(url)
                if input_text==None:
                    await send_response(bot,message,PickRandomResponse(bot['noYTtags']))
                elif input_text=='{[(*VNF*)]}':
                    await send_response(bot,message,PickRandomResponse(bot['URLBroken']))
                else:
                    await send_response(bot,message,f"{PickRandomResponse(bot['YTtags'])}\n\n{input_text}")
                return

            # if the input text starts with "%http", replace it with the actual URL text
            if os.path.exists(bot['TooMuchInformation']) and input_text.lower().strip().startswith('%http'):
                # Web/URL reference
                url=input_text.replace(' ','')[1:]
                input_text=html2text(url)
                if input_text==None:
                    await send_response(bot,message,PickRandomResponse(bot['URLBroken']))
                    return

            # Give the AI the message
            if ResponseLock.acquire(timeout=ResponseTimeout):
                msg={}
                msg['input']=input_text
                msg['gid']=guild.id
                msg['cid']=message.channel.id
                msg['mid']=message.id
                request_list.append(msg)
                ResponseLock.release()
            else:
                ErrorLog("Lock failed OM")
    except discord.HTTPException as err:
        if err.code==429:  # Rate limit error
            retry_after=int(err.response.headers.get('X-RateLimit-Reset', 1))*2
            print(f"Rate limit hit. Retrying after {retry_after} seconds.")
            await asyncio.sleep(retry_after)
            await on_message(message)  # Retry the request
    except Exception as err:
        ErrorLog(f"OM: {sys.exc_info()[-1].tb_lineno}/{str(err)}")

# Start the timer task for responses.

@client.event
async def on_ready():
    print(f'Logged in as {client.user}')

    # Print some fluff
    print("In guilds:")
    for guild in client.guilds:
        mkdir(f'{MemoryStorage}/{guild.id}')
        mkdir(f'{LoggingStorage}/{guild.id}')

        print(f" {guild.name} ({guild.id}), {guild.owner} ({guild.owner.id}), {guild.member_count} members")

        # Check if the bot's role has administrator permissions
        if not guild.me.guild_permissions.administrator:
            print(f"  Error: Companion does not have administrator privileges in {guild.name}.")

    # Start the task when the bot is ready
    update_response_data.start()
    autodelete_messages.start()
    track_active_users.start()
    MemoryMaintenance.start()

    print("Ready to serve!")

###
### STARTUP: starts the bot with the corresponding token
###

if __name__=='__main__':
    print(f'Companion {Version}')
    mkdir(ConfigStorage)

    Tokens=ReadTokens()

    try:
        client.run(Tokens['Discord'],log_handler=None)
    except Exception as err:
        ErrorLog(f"Broken MAIN: {err}")
        print(f"Broken MAIN: {err}")

### END OF PROGRAM