From 66645e87c808e8ba8515b41eb6291e440ded3934 Mon Sep 17 00:00:00 2001 From: Victor Gaydov Date: Tue, 12 Nov 2019 16:52:13 +0300 Subject: [PATCH] feat: add PLC support --- README.md | 12 ++++ decoder.go | 50 ++++++++++++++ opus_test.go | 179 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 241 insertions(+) diff --git a/README.md b/README.md index 1abd727..581803d 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,18 @@ Note regarding Forward Error Correction (FEC): > Note also that in order to use this feature the encoder needs to be configured > with `SetInBandFEC(true)` and `SetPacketLossPerc(x)` options. +Note regarding Packet Loss Concealment (PLC): +> When a packet is considered "lost", `DecodePLC` and `DecodePLCFloat32` methods +> can be called in order to obtain something better sounding than just silence. +> The PCM needs to be exactly the duration of audio that is missing. +> `LastPacketDuration()` can be used on the decoder to get the length of the +> last packet. +> This option does not require any additional encoder options. Unlike FEC, +> PLC does not introduce additional latency. It is calculated from the previous +> packet, not from the next one. +> Note that `DecodeFEC` and `DecodeFECFloat32` automatically fall back to PLC +> when no FEC data is available in the provided packet. + ### Streams (and Files) To decode a .opus file (or .ogg with Opus data), or to decode a "Opus stream" diff --git a/decoder.go b/decoder.go index f656d60..c80b88c 100644 --- a/decoder.go +++ b/decoder.go @@ -178,6 +178,56 @@ func (dec *Decoder) DecodeFECFloat32(data []byte, pcm []float32) error { return nil } +// DecodePLC recovers a lost packet using Opus Packet Loss Concealment feature. +// The supplied buffer needs to be exactly the duration of audio that is missing. +func (dec *Decoder) DecodePLC(pcm []int16) error { + if dec.p == nil { + return errDecUninitialized + } + if len(pcm) == 0 { + return fmt.Errorf("opus: target buffer empty") + } + if cap(pcm)%dec.channels != 0 { + return fmt.Errorf("opus: output buffer capacity must be multiple of channels") + } + n := int(C.opus_decode( + dec.p, + nil, + 0, + (*C.opus_int16)(&pcm[0]), + C.int(cap(pcm)/dec.channels), + 0)) + if n < 0 { + return Error(n) + } + return nil +} + +// DecodePLCFloat32 recovers a lost packet using Opus Packet Loss Concealment feature. +// The supplied buffer needs to be exactly the duration of audio that is missing. +func (dec *Decoder) DecodePLCFloat32(pcm []float32) error { + if dec.p == nil { + return errDecUninitialized + } + if len(pcm) == 0 { + return fmt.Errorf("opus: target buffer empty") + } + if cap(pcm)%dec.channels != 0 { + return fmt.Errorf("opus: output buffer capacity must be multiple of channels") + } + n := int(C.opus_decode_float( + dec.p, + nil, + 0, + (*C.float)(&pcm[0]), + C.int(cap(pcm)/dec.channels), + 0)) + if n < 0 { + return Error(n) + } + return nil +} + // LastPacketDuration gets the duration (in samples) // of the last packet successfully decoded or concealed. func (dec *Decoder) LastPacketDuration() (int, error) { diff --git a/opus_test.go b/opus_test.go index 30f95e2..82e30f0 100644 --- a/opus_test.go +++ b/opus_test.go @@ -256,6 +256,185 @@ func TestCodecFECFloat32(t *testing.T) { */ } +func TestCodecPLC(t *testing.T) { + // Create bogus input sound + const G4 = 391.995 + const SAMPLE_RATE = 48000 + const FRAME_SIZE_MS = 10 + const FRAME_SIZE = SAMPLE_RATE * FRAME_SIZE_MS / 1000 + const NUMBER_OF_FRAMES = 6 + pcm := make([]int16, 0) + + enc, err := NewEncoder(SAMPLE_RATE, 1, AppVoIP) + if err != nil || enc == nil { + t.Fatalf("Error creating new encoder: %v", err) + } + + dec, err := NewDecoder(SAMPLE_RATE, 1) + if err != nil || dec == nil { + t.Fatalf("Error creating new decoder: %v", err) + } + + mono := make([]int16, FRAME_SIZE*NUMBER_OF_FRAMES) + addSine(mono, SAMPLE_RATE, G4) + + encodedData := make([][]byte, NUMBER_OF_FRAMES) + for i, idx := 0, 0; i < len(mono); i += FRAME_SIZE { + data := make([]byte, 1000) + n, err := enc.Encode(mono[i:i+FRAME_SIZE], data) + if err != nil { + t.Fatalf("Couldn't encode data: %v", err) + } + data = data[:n] + encodedData[idx] = data + idx++ + } + + lost := false + for i := 0; i < len(encodedData); i++ { + // "Dropping" packets 2 and 4 + if i == 2 || i == 4 { + lost = true + continue + } + if lost { + samples, err := dec.LastPacketDuration() + if err != nil { + t.Fatalf("Couldn't get last packet duration: %v", err) + } + + pcmBuffer := make([]int16, samples) + if err = dec.DecodePLC(pcmBuffer); err != nil { + t.Fatalf("Couldn't decode data: %v", err) + } + nonZero := 0 + for n := range pcmBuffer { + if pcmBuffer[n] != 0 { + nonZero++ + } + } + if nonZero == 0 { + t.Fatalf("DecodePLC produced only zero samples") + } + pcm = append(pcm, pcmBuffer...) + lost = false + } + + pcmBuffer := make([]int16, NUMBER_OF_FRAMES*FRAME_SIZE) + n, err := dec.Decode(encodedData[i], pcmBuffer) + if err != nil { + t.Fatalf("Couldn't decode data: %v", err) + } + pcmBuffer = pcmBuffer[:n] + if n != FRAME_SIZE { + t.Fatalf("Length mismatch: %d samples expected, %d out", FRAME_SIZE, n) + } + pcm = append(pcm, pcmBuffer...) + } + + if len(mono) != len(pcm) { + t.Fatalf("Input/Output length mismatch: %d samples in, %d out", len(mono), len(pcm)) + } + + // Commented out for the same reason as in TestStereo + /* + fmt.Printf("in,out\n") + for i := range mono { + fmt.Printf("%d,%d\n", mono[i], pcm[i]) + } + */ + +} + +func TestCodecPLCFloat32(t *testing.T) { + // Create bogus input sound + const G4 = 391.995 + const SAMPLE_RATE = 48000 + const FRAME_SIZE_MS = 10 + const FRAME_SIZE = SAMPLE_RATE * FRAME_SIZE_MS / 1000 + const NUMBER_OF_FRAMES = 6 + pcm := make([]float32, 0) + + enc, err := NewEncoder(SAMPLE_RATE, 1, AppVoIP) + if err != nil || enc == nil { + t.Fatalf("Error creating new encoder: %v", err) + } + + dec, err := NewDecoder(SAMPLE_RATE, 1) + if err != nil || dec == nil { + t.Fatalf("Error creating new decoder: %v", err) + } + + mono := make([]float32, FRAME_SIZE*NUMBER_OF_FRAMES) + addSineFloat32(mono, SAMPLE_RATE, G4) + + encodedData := make([][]byte, NUMBER_OF_FRAMES) + for i, idx := 0, 0; i < len(mono); i += FRAME_SIZE { + data := make([]byte, 1000) + n, err := enc.EncodeFloat32(mono[i:i+FRAME_SIZE], data) + if err != nil { + t.Fatalf("Couldn't encode data: %v", err) + } + data = data[:n] + encodedData[idx] = data + idx++ + } + + lost := false + for i := 0; i < len(encodedData); i++ { + // "Dropping" packets 2 and 4 + if i == 2 || i == 4 { + lost = true + continue + } + if lost { + samples, err := dec.LastPacketDuration() + if err != nil { + t.Fatalf("Couldn't get last packet duration: %v", err) + } + + pcmBuffer := make([]float32, samples) + if err = dec.DecodePLCFloat32(pcmBuffer); err != nil { + t.Fatalf("Couldn't decode data: %v", err) + } + nonZero := 0 + for n := range pcmBuffer { + if pcmBuffer[n] != 0.0 { + nonZero++ + } + } + if nonZero == 0 { + t.Fatalf("DecodePLC produced only zero samples") + } + pcm = append(pcm, pcmBuffer...) + lost = false + } + + pcmBuffer := make([]float32, NUMBER_OF_FRAMES*FRAME_SIZE) + n, err := dec.DecodeFloat32(encodedData[i], pcmBuffer) + if err != nil { + t.Fatalf("Couldn't decode data: %v", err) + } + pcmBuffer = pcmBuffer[:n] + if n != FRAME_SIZE { + t.Fatalf("Length mismatch: %d samples expected, %d out", FRAME_SIZE, n) + } + pcm = append(pcm, pcmBuffer...) + } + + if len(mono) != len(pcm) { + t.Fatalf("Input/Output length mismatch: %d samples in, %d out", len(mono), len(pcm)) + } + + // Commented out for the same reason as in TestStereo + /* + fmt.Printf("in,out\n") + for i := 0; i < len(mono); i++ { + fmt.Printf("%f,%f\n", mono[i], pcm[i]) + } + */ +} + func TestStereo(t *testing.T) { // Create bogus input sound const G4 = 391.995 -- 2.48.1