los_angeles_music_composer_edition.py

# -*- coding: utf-8 -*-
"""Los_Angeles_Music_Composer_Edition.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/github/asigalov61/Los-Angeles-Music-Composer/blob/main/Los_Angeles_Music_Composer_Edition.ipynb

# Los Angeles Music Composer Edition (ver. 1.1)

***

Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools

***

WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/

***

#### Project Los Angeles

#### Tegridy Code 2023

***

# (GPU CHECK)
"""

#@title NVIDIA GPU check
!nvidia-smi

"""# (SETUP ENVIRONMENT)"""

#@title Install dependencies
!git clone https://github.com/asigalov61/Los-Angeles-Music-Composer
!pip install torch
!pip install einops
!pip install torch-summary
!pip install sklearn
!pip install tqdm
!pip install matplotlib
!apt install fluidsynth #Pip does not work for some reason. Only apt works
!pip install midi2audio

# Commented out IPython magic to ensure Python compatibility.
#@title Import modules

print('=' * 70)
print('Loading core Los Angeles Music Composer modules...')

import os
import pickle
import random
import secrets
import statistics
from time import time
import tqdm

print('=' * 70)
print('Loading main Los Angeles Music Composer modules...')
import torch

# %cd /content/Los-Angeles-Music-Composer

import TMIDIX

from lwa_transformer import *

# %cd /content/
print('=' * 70)
print('Loading aux Los Angeles Music Composer modeules...')

import matplotlib.pyplot as plt

from torchsummary import summary
from sklearn import metrics

from midi2audio import FluidSynth
from IPython.display import Audio, display

print('=' * 70)
print('Done!')
print('Enjoy! :)')
print('=' * 70)

"""# (LOAD MODEL)"""

# Commented out IPython magic to ensure Python compatibility.
#@title Unzip Pre-Trained Los Angeles Music Composer Model
print('=' * 70)
# %cd /content/Los-Angeles-Music-Composer/Model

print('=' * 70)
print('Unzipping pre-trained Los Angeles Music Composer model...Please wait...')

!cat /content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Trained_Model.zip* > /content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Trained_Model.zip
print('=' * 70)

!unzip -j /content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Trained_Model.zip
print('=' * 70)

print('Done! Enjoy! :)')
print('=' * 70)
# %cd /content/
print('=' * 70)

#@title Load Los Angeles Music Composer Model
full_path_to_model_checkpoint = "/content/Los-Angeles-Music-Composer/Model/Los_Angeles_Music_Composer_Model_88835_steps_0.643_loss.pth" #@param {type:"string"}

print('=' * 70)
print('Loading Los Angeles Music Composer Pre-Trained Model...')
print('Please wait...')
print('=' * 70)
print('Instantiating model...')

SEQ_LEN = 4096

# instantiate the model

model = LocalTransformer(
    num_tokens = 2831,
    dim = 1024,
    depth = 36,
    causal = True,
    local_attn_window_size = 512,
    max_seq_len = SEQ_LEN
)

model = torch.nn.DataParallel(model)

model.cuda()
print('=' * 70)

print('Loading model checkpoint...')

model.load_state_dict(torch.load(full_path_to_model_checkpoint))
print('=' * 70)

model.eval()

print('Done!')
print('=' * 70)

# Model stats
print('Model summary...')
summary(model)

# Plot Token Embeddings
tok_emb = model.module.token_emb.weight.detach().cpu().tolist()

tok_emb1 = []

for t in tok_emb:
    tok_emb1.append([abs(statistics.median(t))])

cos_sim = metrics.pairwise_distances(
   tok_emb1, metric='euclidean'
)
plt.figure(figsize=(7, 7))
plt.imshow(cos_sim, cmap="inferno", interpolation="nearest")
im_ratio = cos_sim.shape[0] / cos_sim.shape[1]
plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)
plt.xlabel("Position")
plt.ylabel("Position")
plt.tight_layout()
plt.plot()
plt.savefig("/content/Los-Angeles-Music-Composer-Tokens-Embeddings-Plot.png", bbox_inches="tight")

"""# (LOAD SEED MIDI)"""

#@title Load Seed MIDI
select_seed_MIDI = "Los-Angeles-Music-Composer-Piano-Seed-1" #@param ["Los-Angeles-Music-Composer-Piano-Seed-1", "Los-Angeles-Music-Composer-Piano-Seed-2", "Los-Angeles-Music-Composer-Piano-Seed-3", "Los-Angeles-Music-Composer-Piano-Seed-4", "Los-Angeles-Music-Composer-Piano-Seed-5", "Los-Angeles-Music-Composer-MI-Seed-1", "Los-Angeles-Music-Composer-MI-Seed-2", "Los-Angeles-Music-Composer-MI-Seed-3", "Los-Angeles-Music-Composer-MI-Seed-4", "Los-Angeles-Music-Composer-MI-Seed-5"]
full_path_to_custom_seed_MIDI = "" #@param {type:"string"}
number_of_prime_tokens = 402 #@param {type:"slider", min:126, max:3000, step:3}

if full_path_to_custom_seed_MIDI == '':
  f = '/content/Los-Angeles-Music-Composer/Seeds/'+select_seed_MIDI+'.mid'

else:
  f = full_path_to_custom_seed_MIDI

print('=' * 70)
print('Los Angeles Music Composer Seed MIDI Loader')
print('=' * 70)
print('Loading seed MIDI...')
print('=' * 70)
print('File:', f)
print('=' * 70)

#=======================================================
# START PROCESSING

# Convering MIDI to ms score with MIDI.py module
score = TMIDIX.midi2ms_score(open(f, 'rb').read())

# INSTRUMENTS CONVERSION CYCLE
events_matrix = []
melody_chords_f = []
melody_chords_f1 = []
itrack = 1
patches = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

patch_map = [
            [0, 1, 2, 3, 4, 5, 6, 7], # Piano 
            [24, 25, 26, 27, 28, 29, 30], # Guitar
            [32, 33, 34, 35, 36, 37, 38, 39], # Bass
            [40, 41], # Violin
            [42, 43], # Cello
            [46], # Harp
            [56, 57, 58, 59, 60], # Trumpet
            [64, 65, 66, 67, 68, 69, 70, 71], # Sax
            [72, 73, 74, 75, 76, 77, 78], # Flute
            [-1], # Drums
            [52, 53], # Choir
            [16, 17, 18, 19, 20] # Organ
            ]

while itrack < len(score):
  for event in score[itrack]:         
      if event[0] == 'note' or event[0] == 'patch_change':
          events_matrix.append(event)
  itrack += 1

events_matrix.sort(key=lambda x: x[1])

events_matrix1 = []

for event in events_matrix:
  if event[0] == 'patch_change':
      patches[event[2]] = event[3]

  if event[0] == 'note':
      event.extend([patches[event[3]]])
      once = False

      for p in patch_map:
          if event[6] in p and event[3] != 9: # Except the drums
              event[3] = patch_map.index(p)
              once = True

      if not once and event[3] != 9: # Except the drums
          event[3] = 15 # All other instruments/patches channel
          event[5] = max(80, event[5])

      if event[3] < 12: # We won't write chans 12-16 for now...
          events_matrix1.append(event)
          # stats[event[3]] += 1

#=======================================================
# PRE-PROCESSING

# checking number of instruments in a composition
instruments_list_without_drums = list(set([y[3] for y in events_matrix1 if y[3] != 9]))

if len(events_matrix1) > 0 and len(instruments_list_without_drums) > 0:

  # recalculating timings
  for e in events_matrix1:
      e[1] = int(e[1] / 10) # Max 1 seconds for start-times
      e[2] = int(e[2] / 20) # Max 2 seconds for durations

  # Sorting by pitch, then by start-time
  events_matrix1.sort(key=lambda x: x[4], reverse=True)
  events_matrix1.sort(key=lambda x: x[1])

  #=======================================================
  # FINAL PRE-PROCESSING

  melody_chords = []

  pe = events_matrix1[0]

  for e in events_matrix1:
    if e[1] >= 0 and e[2] > 0:

      # Cliping all values...
      tim = max(0, min(127, e[1]-pe[1]))             
      dur = max(1, min(127, e[2]))
      cha = max(0, min(11, e[3]))
      ptc = max(1, min(127, e[4]))
      vel = max(8, min(127, e[5]))

      velocity = round(vel / 15)

      # Writing final note 
      melody_chords.append([tim, dur, cha, ptc, velocity])

      pe = e



instruments_list = list(set([y[2] for y in melody_chords]))
num_instr = len(instruments_list)

#=======================================================
# FINAL PROCESSING
#=======================================================

# Break between compositions / Intro seq

if 9 in instruments_list:
  drums_present = 2818 # Yes
else:
  drums_present = 2817 # No

melody_chords_f.extend([2816, drums_present, 2819+(num_instr-1)])

#=======================================================

# Composition control seq
intro_mode_time = statistics.mode([0] + [y[0] for y in melody_chords if y[2] != 9 and y[0] != 0])
intro_mode_dur = statistics.mode([y[1] for y in melody_chords if y[2] != 9])
intro_mode_pitch = statistics.mode([y[3] for y in melody_chords if y[2] != 9])
intro_mode_velocity = statistics.mode([y[4] for y in melody_chords if y[2] != 9])

# Instrument value 12 is reserved for composition control seq
intro_dur_vel = (intro_mode_dur * 8) + (intro_mode_velocity-1)
intro_cha_ptc = (12 * 128) + intro_mode_pitch

melody_chords_f.extend([intro_mode_time, intro_dur_vel+128, intro_cha_ptc+1152])

# TOTAL DICTIONARY SIZE 2831

#=======================================================
# MAIN PROCESSING CYCLE
#=======================================================

for m in melody_chords:

  # WRITING EACH NOTE HERE
  dur_vel = (m[1] * 8) + (m[4]-1)
  cha_ptc = (m[2] * 128) + m[3]

  melody_chords_f.extend([m[0], dur_vel+128, cha_ptc+1152])
  melody_chords_f1.append([m[0], dur_vel+128, cha_ptc+1152])

melody_chords_f1 = melody_chords_f1[:(number_of_prime_tokens // 3)]
melody_chords_f = melody_chords_f[:number_of_prime_tokens]

#=======================================================
  
song = melody_chords_f
song_f = []
tim = 0
dur = 0
vel = 0
pitch = 0
channel = 0

son = []
song1 = []

for s in song:
  if s >= 128 and s < (12*128)+1152:
    son.append(s)
  else:
    if len(son) == 3:
      song1.append(son)
    son = []
    son.append(s)
                
for ss in song1:

  tim += ss[0] * 10

  dur = ((ss[1]-128) // 8) * 20
  vel = (((ss[1]-128) % 8)+1) * 15

  channel = (ss[2]-1152) // 128
  pitch = (ss[2]-1152) % 128
                  
  song_f.append(['note', tim, dur, channel, pitch, vel ])

detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,
                                                      output_signature = 'Los Angeles Music Composer',  
                                                      output_file_name = '/content/Los-Angeles-Music-Composer-Seed-Composition',
                                                      track_name='Project Los Angeles',
                                                      list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 19, 0, 0, 0, 0],
                                                      number_of_ticks_per_quarter=500)
    
#=======================================================

print('=' * 70)
print('Composition stats:')
print('Composition has', len(melody_chords_f1), 'notes')
print('Composition has', len(melody_chords_f), 'tokens')
print('=' * 70)

print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/Los-Angeles-Music-Composer-Seed-Composition'

x = []
y =[]
c = []

colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']

block_lines = [(song_f[-1][1] / 1000)]

for s in song_f:
  x.append(s[1] / 1000)
  y.append(s[4])
  c.append(colors[s[3]])

FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
display(Audio(str(fname + '.wav'), rate=16000))

plt.figure(figsize=(14,5))
ax=plt.axes(title=fname)
ax.set_facecolor('black')

plt.scatter(x,y, c=c)
plt.xlabel("Time")
plt.ylabel("Pitch")
plt.show()

"""# (COMPOSITION LOOP)

## Run the cells below in a loop to generate endless continuation
"""

#@title Standard/Simple Continuation

number_of_batches_to_generate = 4 #@param {type:"slider", min:1, max:16, step:1}
number_of_memory_tokens = 4095 #@param {type:"slider", min:402, max:4095, step:3}
temperature = 1 #@param {type:"slider", min:0.1, max:1, step:0.1}

print('=' * 70)
print('Los Angeles Music Composer Standard Model Generator')
print('=' * 70)

preview = melody_chords_f[-120:]

inp = [melody_chords_f[-number_of_memory_tokens:]] * number_of_batches_to_generate

inp = torch.LongTensor(inp).cuda()

start_time = time()

out = model.module.generate(inp, 
                            402, 
                            temperature=temperature, 
                            return_prime=False,  
                            verbose=True)

out0 = out.tolist()

print('=' * 70)
print('Done!')

print('=' * 70)
print('Generation took', time() - start_time, "seconds")

#======================================================================
print('=' * 70)
print('Rendering results...')

for i in range(number_of_batches_to_generate):

  print('=' * 70)
  print('Batch #', i)
  print('=' * 70)

  out1 = out0[i]

  print('Sample INTs', out1[:12])
  print('=' * 70)

  if len(out) != 0:
      
      song = preview + out1
      song_f = []
      tim = 0
      dur = 0
      vel = 0
      pitch = 0
      channel = 0

      son = []
      song1 = []

      for s in song:
        if s >= 128 and s < (12*128)+1152:
          son.append(s)
        else:
          if len(son) == 3:
            song1.append(son)
          son = []
          son.append(s)
                      
      for ss in song1:

        tim += ss[0] * 10

        dur = ((ss[1]-128) // 8) * 20
        vel = (((ss[1]-128) % 8)+1) * 15
    
        channel = (ss[2]-1152) // 128
        pitch = (ss[2]-1152) % 128
                        
        song_f.append(['note', tim, dur, channel, pitch, vel ])

      detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,
                                                          output_signature = 'Los Angeles Music Composer',  
                                                          output_file_name = '/content/Los-Angeles-Music-Composer-Music-Composition_'+str(i), 
                                                          track_name='Project Los Angeles',
                                                          list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 19, 0, 0, 0, 0],
                                                          number_of_ticks_per_quarter=500)
      print('=' * 70)
      print('Displaying resulting composition...')
      print('=' * 70)

      fname = '/content/Los-Angeles-Music-Composer-Music-Composition_'+str(i)

      x = []
      y =[]
      c = []

      colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']

      for s in song_f:
        x.append(s[1] / 1000)
        y.append(s[4])
        c.append(colors[s[3]])

      FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
      display(Audio(str(fname + '.wav'), rate=16000))

      plt.figure(figsize=(14,5))
      ax=plt.axes(title=fname)
      ax.set_facecolor('black')

      plt.scatter(x,y, c=c)

      pbl = song_f[39][1] / 1000

      ax.axvline(x=pbl, c='w')

      plt.xlabel("Time")
      plt.ylabel("Pitch")
      plt.show()

#@title Choose one generated block to add to the composition
block_action = "add_last_generated_block" #@param ["add_last_generated_block", "remove_last_added_block"]
add_block_with_batch_number = 0 #@param {type:"slider", min:0, max:15, step:1}

print('=' * 70)

if block_action == 'add_last_generated_block':
  melody_chords_f.extend(out0[min(len(out0)-1, add_block_with_batch_number)])
  print('Block added!')
else:
  melody_chords_f = melody_chords_f[:max(number_of_prime_tokens, (len(melody_chords_f)-402))]
  print('Block removed!')

print('=' * 70)
print('Composition now has', (len(melody_chords_f) // 4), 'notes')
print('Composition now has', len(melody_chords_f), 'tokens')


print('=' * 70)
print('Sample INTs', out1[:12])
print('=' * 70)

if len(melody_chords_f) != 0:
    
    song = melody_chords_f
    song_f = []
    tim = 0
    dur = 0
    vel = 0
    pitch = 0
    channel = 0

    son = []
    song1 = []

    for s in song:
      if s >= 128 and s < (12*128)+1152:
        son.append(s)
      else:
        if len(son) == 3:
          song1.append(son)
        son = []
        son.append(s)
                    
    for ss in song1:

      tim += ss[0] * 10

      dur = ((ss[1]-128) // 8) * 20
      vel = (((ss[1]-128) % 8)+1) * 15
  
      channel = (ss[2]-1152) // 128
      pitch = (ss[2]-1152) % 128
                      
      song_f.append(['note', tim, dur, channel, pitch, vel ])

    detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,
                                                        output_signature = 'Los Angeles Music Composer',  
                                                        output_file_name = '/content/Los-Angeles-Music-Composer-Music-Composition', 
                                                        track_name='Project Los Angeles',
                                                        list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 19, 0, 0, 0, 0],
                                                        number_of_ticks_per_quarter=500)
    print('=' * 70)
    print('Displaying resulting composition...')
    print('=' * 70)

    fname = '/content/Los-Angeles-Music-Composer-Music-Composition'

    x = []
    y =[]
    c = []

    colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']
    
    if block_action == 'add_last_generated_block':
      block_lines.append((song_f[-1][1] / 1000))
    else:
      if len(block_lines) > 1:
        block_lines.pop()


    for s in song_f:
      x.append(s[1] / 1000)
      y.append(s[4])
      c.append(colors[s[3]])

    FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
    display(Audio(str(fname + '.wav'), rate=16000))

    plt.figure(figsize=(14,5))
    ax=plt.axes(title=fname)
    ax.set_facecolor('black')

    plt.scatter(x,y, c=c)

    for bl in block_lines:
      ax.axvline(x=bl, c='w')

    plt.xlabel("Time")
    plt.ylabel("Pitch")
    plt.show()

"""# Congrats! You did it! :)"""