From a3aa9ce4ea89392033bc7b04b235fa7d16ea170a Mon Sep 17 00:00:00 2001 From: tremblap Date: Fri, 1 Mar 2024 17:42:07 +0000 Subject: [PATCH] added multi-description example with dataset and json output (#260) * added multi-description example with dataset and json output * corrected readme * example with UMAP in action * Update flucoma-core for hisstools header-only * Update hisstools library tag * clang_format the relevant files * Update HISSTools tags for out audio file fixes * Update tag for hisstools_library for sample writing offset fix * clang-formatted --------- Co-authored-by: lewardo Co-authored-by: FluCoMa Co-authored-by: Alex Harker --- examples/CMakeLists.txt | 16 ++-- examples/README.md | 6 +- examples/dataset.cpp | 162 ++++++++++++++++++++++++++++++++++++++++ examples/umap.cpp | 76 +++++++++++++++++++ 4 files changed, 249 insertions(+), 11 deletions(-) create mode 100644 examples/dataset.cpp create mode 100644 examples/umap.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 120dd794e..6ba53f77d 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,13 +1,9 @@ -foreach (EXAMPLE describe) +foreach (EXAMPLE dataset describe umap) - add_executable ( - ${EXAMPLE} ${EXAMPLE}.cpp - ) +add_executable(${EXAMPLE} ${EXAMPLE}.cpp) - target_link_libraries( - ${EXAMPLE} PRIVATE FLUID_DECOMPOSITION - ) + target_link_libraries(${EXAMPLE} PRIVATE FLUID_DECOMPOSITION) - target_compile_options(${EXAMPLE} PRIVATE ${FLUID_ARCH}) - -endforeach (EXAMPLE) + target_compile_options(${EXAMPLE} PRIVATE ${FLUID_ARCH}) + + endforeach(EXAMPLE) diff --git a/examples/README.md b/examples/README.md index 0466991dd..83cd02b8b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,7 +9,11 @@ mkdir build && cd build # CMake to configure targets cmake .. -DBUILD_EXAMPLES=ON -# build example target, currently only `describe` +# build an example target, currently three options: +# 'describe' takes a single file in and prints stats on various descriptors +# 'dataset' takes a list of soundfiles and make an entry per file in a dataset saved as json +# 'umap' takes a multidimensional dataset input as json, and will use UMAP to reduce the dimension count to 2 + cmake --build . --target describe # OR make describe diff --git a/examples/dataset.cpp b/examples/dataset.cpp new file mode 100644 index 000000000..8328a10d3 --- /dev/null +++ b/examples/dataset.cpp @@ -0,0 +1,162 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +/* +This program demonstrates the use of the fluid decomposition toolbox +to produces a dataset with the summary of spectral features on files +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +fluid::RealVector computeStats(fluid::RealMatrixView matrix, + fluid::algorithm::MultiStats stats) +{ + fluid::index dim = matrix.cols(); + fluid::RealMatrix tmp(dim, 7); + fluid::RealVector result(dim * 7); + stats.process(matrix.transpose(), tmp); + for (int j = 0; j < dim; j++) + { + result(fluid::Slice(j * 7, 7)) <<= tmp.row(j); + } + return result; +} + +int main(int argc, char* argv[]) +{ + using namespace fluid; + using namespace fluid::algorithm; + using fluid::index; + + if (argc <= 2) + { + std::cerr << "usage: describe output_file.json input_file_1.wav " + "input_file_2.wav...\n"; + return 1; + } + + FluidDataSet dataset(168); + + const char* outputFile = argv[1]; + for (int i = 2; i < argc; i++) + { + const char* inputFile = argv[i]; + htl::in_audio_file file(inputFile); + + index nSamples = file.frames(); + auto samplingRate = file.sampling_rate(); + + if (!file.is_open()) + { + std::cerr << "input file " << inputFile << " could not be opened\n"; + return -2; + } + + index nBins = 513; + index fftSize = 2 * (nBins - 1); + index hopSize = 1024; + index windowSize = 1024; + index halfWindow = windowSize / 2; + index nBands = 40; + index nCoefs = 13; + index minFreq = 20; + index maxFreq = 5000; + + STFT stft{windowSize, fftSize, hopSize}; + MelBands bands{nBands, fftSize}; + DCT dct{nBands, nCoefs}; + YINFFT yin{nBins, FluidDefaultAllocator()}; + SpectralShape shape(FluidDefaultAllocator()); + Loudness loudness{windowSize}; + MultiStats stats; + + bands.init(minFreq, maxFreq, nBands, nBins, samplingRate, windowSize); + dct.init(nBands, nCoefs); + stats.init(0, 0, 50, 100); + loudness.init(windowSize, samplingRate); + + RealVector in(nSamples); + file.read_channel(in.data(), nSamples, 0); + RealVector padded(in.size() + windowSize + hopSize); + index nFrames = floor((padded.size() - windowSize) / hopSize); + RealMatrix pitchMat(nFrames, 2); + RealMatrix loudnessMat(nFrames, 2); + RealMatrix mfccMat(nFrames, nCoefs); + RealMatrix shapeMat(nFrames, 7); + std::fill(padded.begin(), padded.end(), 0); + padded(Slice(halfWindow, in.size())) <<= in; + + for (int i = 0; i < nFrames; i++) + { + ComplexVector frame(nBins); + RealVector magnitude(nBins); + RealVector mels(nBands); + RealVector mfccs(nCoefs); + RealVector pitch(2); + RealVector shapeDesc(7); + RealVector loudnessDesc(2); + RealVectorView window = padded(fluid::Slice(i * hopSize, windowSize)); + stft.processFrame(window, frame); + stft.magnitude(frame, magnitude); + bands.processFrame(magnitude, mels, false, false, true, + FluidDefaultAllocator()); + dct.processFrame(mels, mfccs); + mfccMat.row(i) <<= mfccs; + yin.processFrame(magnitude, pitch, minFreq, maxFreq, samplingRate); + pitchMat.row(i) <<= pitch; + shape.processFrame(magnitude, shapeDesc, samplingRate, 0, -1, 0.95, false, + false, FluidDefaultAllocator()); + shapeMat.row(i) <<= shapeDesc; + loudness.processFrame(window, loudnessDesc, true, true); + loudnessMat.row(i) <<= loudnessDesc; + } + + RealVector pitchStats = computeStats(pitchMat, stats); + RealVector loudnessStats = computeStats(loudnessMat, stats); + RealVector shapeStats = computeStats(shapeMat, stats); + RealVector mfccStats = computeStats(mfccMat, stats); + + RealVector allStats(168); + + allStats(fluid::Slice(0, 14)) <<= pitchStats; + allStats(fluid::Slice(14, 14)) <<= loudnessStats; + allStats(fluid::Slice(28, 49)) <<= shapeStats; + allStats(fluid::Slice(77, 91)) <<= mfccStats; + + dataset.add(inputFile, allStats); + } + + auto outputJSON = JSONFile(outputFile, "w"); + outputJSON.write(dataset); + + if (!outputJSON.ok()) + { + std::cerr << "failed to write output to " << outputFile << "\n"; + } + + return 0; +} diff --git a/examples/umap.cpp b/examples/umap.cpp new file mode 100644 index 000000000..56023ab91 --- /dev/null +++ b/examples/umap.cpp @@ -0,0 +1,76 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +/* +This program demonstrates the use of the fluid decomposition toolbox +to apply an algorithm on an input dataset +*/ + +#include "algorithms/public/UMAP.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char* argv[]) +{ + using namespace fluid; + using namespace fluid::algorithm; + using fluid::index; + + if (argc != 3) + { + std::cerr << "usage: umap input_file.json output_file.json\n"; + return 1; + } + + FluidDataSet datasetIN(1); + FluidDataSet datasetOUT(1); + + const char* inputFile = argv[1]; + const char* outputFile = argv[2]; + + auto inputJSON = JSONFile(inputFile, "r"); + nlohmann::json j = inputJSON.read(); + + if (!inputJSON.ok()) + { + std::cerr << "failed to read input " << inputFile << "\n"; + return 2; + } + + if (!check_json(j, datasetIN)) + { + std::cerr << "Invalid JSON format\n"; + return 3; + } + + datasetIN = j.get>(); + + algorithm::UMAP algorithm; + + datasetOUT = algorithm.train(datasetIN, 15, 2, 0.1, 200, 0.1); + + auto outputJSON = JSONFile(outputFile, "w"); + outputJSON.write(datasetOUT); + + if (!outputJSON.ok()) + { + std::cerr << "failed to write output to " << outputFile << "\n"; + } + + return 0; +}