From c16b38d86c71ea4829c6fc85aa2553c8cac406a2 Mon Sep 17 00:00:00 2001 From: mirko Date: Tue, 2 Apr 2019 10:25:32 +0200 Subject: [PATCH 01/22] remove bold in formulas --- src/expectation_propagation.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/expectation_propagation.jl b/src/expectation_propagation.jl index 59613e1..868e420 100644 --- a/src/expectation_propagation.jl +++ b/src/expectation_propagation.jl @@ -58,13 +58,13 @@ end EP for approximate inference of -``P(\\bf{x})=\\frac1Z exp(-\\frac12\\bf{x}' A \\bf{x} + \\bf{x'} \\bf{y}))×\\prod_i p_{i}(x_i)`` +``P( \\bf{x} )=\\frac1Z exp(-\\frac12\\bf{x}' A \\bf{x} + \\bf{x'} \\bf{y}))×\\prod_i p_{i}(x_i)`` Arguments: * `A::Array{Term{T}}`: Gaussian Term (involving only x) * `P0::Array{Prior}`: Prior terms (involving x and y) -* `F::AbstractMatrix{T}`: If included, the unknown becomes ``(\\bf{x},\\bf{y})^T`` and a term ``\\delta(F \\bf{x}+\\bf{d}-\\bf{y})`` is added. +* `F::AbstractMatrix{T}`: If included, the unknown becomes ``(\\bf{x} ,\\bf{y} )^T`` and a term ``\\delta(F \\bf{x}+\\bf{d}-\\bf{y})`` is added. Optional named arguments: From b4436b8858fffd95ad539fca106bb2db2550c53f Mon Sep 17 00:00:00 2001 From: mirko Date: Tue, 2 Apr 2019 10:36:40 +0200 Subject: [PATCH 02/22] fix log in docs --- src/Term.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Term.jl b/src/Term.jl index 2c2db88..dfb048e 100644 --- a/src/Term.jl +++ b/src/Term.jl @@ -5,7 +5,7 @@ This type represents an interaction term in the energy function of the form The complete energy function is given by -``∑_i β_i (\\frac12 x' A_i x + x' y_i + c_i) + M_i log β_i`` +``∑_i β_i (\\frac12 x' A_i x + x' y_i + c_i) + M_i \\log β_i`` as is represented by an Vector{Term}. Note that c and M are only needed for paramenter learning """ From 9b409bc840fe3aaddc22ae5a364be4a2178359af Mon Sep 17 00:00:00 2001 From: mirko Date: Thu, 4 Apr 2019 23:00:29 +0200 Subject: [PATCH 03/22] displaying my own documentation --- docs/make.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/make.jl b/docs/make.jl index 156d905..c455018 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,6 +6,6 @@ makedocs(sitename = "GaussianEP", doctest = true) deploydocs( branch = "gh-pages", - repo = "github.com/abraunst/GaussianEP.git", + repo = "github.com/mpieropan/GaussianEP.git", versions = ["stable" => "v^"] ) From b401859b0fd42d05b1602738495d6f4829d912a7 Mon Sep 17 00:00:00 2001 From: mirko Date: Fri, 12 Apr 2019 13:41:27 +0200 Subject: [PATCH 04/22] restored github.com/abraunst/GaussianEP.git in deploydocs repo --- docs/make.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/make.jl b/docs/make.jl index c455018..156d905 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,6 +6,6 @@ makedocs(sitename = "GaussianEP", doctest = true) deploydocs( branch = "gh-pages", - repo = "github.com/mpieropan/GaussianEP.git", + repo = "github.com/abraunst/GaussianEP.git", versions = ["stable" => "v^"] ) From c06023c2d62d1648c950eb2118f318658ac9654d Mon Sep 17 00:00:00 2001 From: mirko Date: Wed, 17 Apr 2019 22:44:33 +0200 Subject: [PATCH 05/22] Make EPOut mutable --- src/expectation_propagation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/expectation_propagation.jl b/src/expectation_propagation.jl index 868e420..7715a25 100644 --- a/src/expectation_propagation.jl +++ b/src/expectation_propagation.jl @@ -31,7 +31,7 @@ EPState{T}(N, Nx = N) where {T <: AbstractFloat} = EPState{T}(Matrix{T}(undef,Nx Output of EP algorithm """ -struct EPOut{T<:AbstractFloat} +mutable struct EPOut{T<:AbstractFloat} av::Vector{T} va::Vector{T} μ::Vector{T} From 5846e110b83f9a4bb86e61518cc5ae4bfb67562e Mon Sep 17 00:00:00 2001 From: mirko Date: Mon, 17 Aug 2020 22:39:42 +0200 Subject: [PATCH 06/22] added Gaussian and Bernoulli RBM units --- src/priors.jl | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/priors.jl b/src/priors.jl index bd67544..56d2108 100644 --- a/src/priors.jl +++ b/src/priors.jl @@ -289,3 +289,33 @@ function moments(::ThetaPrior,μ,σ) return av,var end +""" +Bernoulli-Bernoulli RBM +""" +struct RBM_Bias_Factor{T<:Real} <: Prior + g::T +end + +function moments(p0::RBM_Bias_Factor,μ,σ) + arg=-p0.g+(1.0-2.0*μ)/(2.0*σ^2.0) + av=1.0/(1.0+exp(arg)) + va=0.5/(1.0+cosh(arg)) + return av,va +end + +""" +Gaussian RBM units +""" +struct RBM_Gaussian_Factor{T<:Real} <: Prior + γ::T + θ::T +end + +function moments(p0::RBM_Gaussian_Factor,μ,σ) + av = (μ+p0.θ*σ^2.0)/(1.0+p0.γ*σ^2.0) + secmom = ((μ+p0.θ*σ^2.0)^2.0+σ^2.0*(1.0+p0.γ*σ^2.0))/(1+p0.γ*σ^2.0)^2.0 + #secmom = (μ^2.0+σ^2.0+2.0*p0.θ*μ*σ^2.0+(p0.γ+p0.θ^2.0)*σ^2.0)/(1+p0.γ*σ^2.0)^2.0 + #va = clamp(secmom-av^2.0,1e-50,1e50) + va=1.0/(1.0/σ^2.0+p0.γ) + return av,va +end From d2e382fa49ab8400e6d85aa9afd27613f3243d69 Mon Sep 17 00:00:00 2001 From: mirko Date: Fri, 25 Sep 2020 22:29:07 +0200 Subject: [PATCH 07/22] Added posdef assertion for matrix A --- src/expectation_propagation.jl | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/expectation_propagation.jl b/src/expectation_propagation.jl index 7715a25..904b02a 100644 --- a/src/expectation_propagation.jl +++ b/src/expectation_propagation.jl @@ -75,7 +75,7 @@ Optional named arguments: * `epsconv::T = 1e-6`: convergence criterion * `maxvar::T = 1e50`: maximum variance * `minvar::T = 1e-50`: minimum variance -* `inverter::Function = inv`: inverter method +* `inverter = inv`: inverter method # Example @@ -95,15 +95,15 @@ julia> res = expectation_propagation([t], P, F) GaussianEP.EPOut{Float64}([0.499997, 0.499997, 3.66527e-15], [0.083325, 0.083325, 0.204301], [0.489862, 0.489862, 3.66599e-15], [334.018, 334.018, 0.204341], :converged, EPState{Float64}([9.79055 -0.00299477; -0.00299477 9.79055], [0.0, 0.0], [0.102139 3.12427e-5; 3.12427e-5 0.102139], [0.489862, 0.489862], [0.499997, 0.499997, 3.66527e-15], [0.083325, 0.083325, 0.204301], [0.490876, 0.490876, -1.86785e-17], [0.489862, 0.489862, 3.66599e-15], [0.100288, 0.100288, 403.599], [334.018, 334.018, 0.204341])) ``` """ -function expectation_propagation(H::Vector{Term{T}}, P0::Vector{P}, F::AbstractMatrix{T} = zeros(T,0,length(P0)), d::AbstractVector{T} = zeros(T,size(F,1)); +function expectation_propagation(H::AbstractVector{Term{T}}, P0::AbstractVector{P}, F::AbstractMatrix{T} = zeros(T,0,length(P0)), d::AbstractVector{T} = zeros(T,size(F,1)); maxiter::Int = 2000, callback = (x...)->nothing, state::EPState{T} = EPState{T}(sum(size(F)), size(F)[2]), - damp::T = 0.9, - epsconv::T = 1e-6, - maxvar::T = 1e50, - minvar::T = 1e-50, - inverter::Function = inv) where {T <: Real, P <: Prior} + damp::T = T(0.9), + epsconv::T = T(1e-6), + maxvar::T = T(1e50), + minvar::T = T(1e-50), + inverter = inv) where {T <: Real, P <: Prior} @extract state A y Σ v av va a μ b s Ny,Nx = size(F) N = Nx + Ny @@ -113,14 +113,16 @@ function expectation_propagation(H::Vector{Term{T}}, P0::Vector{P}, F::AbstractM sum!(A,y,H) Δμ, Δs, Δav, Δva = 0.0, 0.0, 0.0, 0.0 A .+= Diagonal(1 ./ b[1:Nx]) .+ Fp * Diagonal(1 ./ b[Nx+1:end]) * F + @assert isposdef(A) Σ .= inverter(A) - v .= Σ * (y .+ a[1:Nx] ./ b[1:Nx] .+ Fp * ((a[Nx+1:end]-d) ./ b[Nx+1:end])) + ax, bx, ay, by = (@view a[1:Nx]), (@view b[1:Nx]), (@view a[Nx+1:end]), (@view b[Nx+1:end]) + v .= Σ * (y .+ ax ./ bx .+ (Fp * ((ay-d) ./ by))) for i in 1:N if i <= Nx ss = clamp(Σ[i,i], minvar, maxvar) vv = v[i] else - x = Fp[:, i-Nx] + x = @view Fp[:, i-Nx] ss = clamp(dot(x, Σ*x), minvar, maxvar) vv = dot(x, v) + d[i-Nx] end @@ -153,7 +155,7 @@ function expectation_propagation(H::Vector{Term{T}}, P0::Vector{P}, F::AbstractM updateβ(H[i], av[1:Nx]) end callback(av,Δav,epsconv,maxiter,H,P0) - if Δav < epsconv + if Δav < epsconv && norm(F*av[1:Nx]+d-av[Nx+1:end]) < 1e-4 return EPOut(state, :converged) end end From faf1c6667f18cd996690188c8ee7fa037b8c6f7c Mon Sep 17 00:00:00 2001 From: mirko Date: Fri, 9 Oct 2020 15:21:34 +0200 Subject: [PATCH 08/22] changed callback call in expectation_propagation --- src/expectation_propagation.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/expectation_propagation.jl b/src/expectation_propagation.jl index 904b02a..058be87 100644 --- a/src/expectation_propagation.jl +++ b/src/expectation_propagation.jl @@ -127,6 +127,7 @@ function expectation_propagation(H::AbstractVector{Term{T}}, P0::AbstractVector{ vv = dot(x, v) + d[i-Nx] end + @warn "iter $iter, variable $i: ss=$ss, b[$i]=$(b[i])" if ss < b[i] Δs = max(Δs, update_err!(s, i, clamp(1/(1/ss - 1/b[i]), minvar, maxvar))) Δμ = max(Δμ, update_err!(μ, i, s[i] * (vv/ss - a[i]/b[i]))) @@ -154,7 +155,7 @@ function expectation_propagation(H::AbstractVector{Term{T}}, P0::AbstractVector{ for i in 1:length(H) updateβ(H[i], av[1:Nx]) end - callback(av,Δav,epsconv,maxiter,H,P0) + callback(av,Δav,va,Δva,epsconv,maxiter,H,P0) if Δav < epsconv && norm(F*av[1:Nx]+d-av[Nx+1:end]) < 1e-4 return EPOut(state, :converged) end From 44b906f92e4ac1b9de4550d350c99a6cc3da008b Mon Sep 17 00:00:00 2001 From: Mirko Pieropan Date: Fri, 20 Nov 2020 13:49:51 +0100 Subject: [PATCH 09/22] added ThetaMixturePrior --- src/priors.jl | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/src/priors.jl b/src/priors.jl index 56d2108..472bfd9 100644 --- a/src/priors.jl +++ b/src/priors.jl @@ -289,6 +289,77 @@ function moments(::ThetaPrior,μ,σ) return av,var end +""" +A mixture of theta priors: p_0(x)=η*Θ(x)+(1-η)*Θ(-x) +""" +mutable struct ThetaMixturePrior{T<:Real} <: Prior + η::T + δη::T +end + +function theta_mixt_factor(x,η) + if abs(x)<=Inf + f=exp(-0.5*x^2.0)/(η*erfc(-sqrt(0.5)*x)+(1.0-η)*erfc(sqrt(0.5)*x)) + else + println("In theta_mixt_factor sono *qui*!") + if η!=0.0 + f=0.5*exp(-0.5*x^2.0)/η + else + f=sqrt(0.5*π)/(1.0/x-1.0/x^3.0+3.0/x^5.0) + end + end +end + +############## +mutable struct ThetaMixturePrior2{T<:Real} <: Prior + η::T + δη::T + thr::T +end + +function theta_mixt_factor2(x,η,thr) + if abs(x)<=thr + f=exp(-0.5*x^2.0)/(η*erfc(-sqrt(0.5)*x)+(1.0-η)*erfc(sqrt(0.5)*x)) + else + println("In theta_mixt_factor sono *qui*!") + if η!=0.0 + f=0.5*exp(-0.5*x^2.0)/η + else + f=sqrt(0.5*π)/(1.0/x-1.0/x^3.0+3.0/x^5.0) + end + end +end + +function moments(p0::ThetaMixturePrior2,μ,σ) + η=p0.η + α=μ/σ + f=theta_mixt_factor2(α,η,p0.thr) + χ=sqrt(2.0/π)*(2.0*η-1.0)*f + av=μ+σ*χ + va=σ^2.0*(1-χ^2.0)-μ*σ*χ + return av,va +end +############### + +function moments(p0::ThetaMixturePrior,μ,σ) + η=p0.η + α=μ/σ + f=theta_mixt_factor(α,η) + χ=sqrt(2.0/π)*(2.0*η-1.0)*f + av=μ+σ*χ + va=σ^2.0*(1-χ^2.0)-μ*σ*χ + return av,va +end + +function gradient(p0::ThetaMixturePrior,μ,σ,∂𝐹::Union{FreeEnGrad,Nothing}) + η=p0.η + x=μ/σ/sqrt(2) + num=2*erf(x) + den=η*erfc(-x)+(1-η)*erfc(x) + p0.η+=p0.δη*num/den + p0.η=clamp(p0.η,0,1) +end + """ Bernoulli-Bernoulli RBM """ From 1d68ab93e24e0695c715f76d16e285e088b96413 Mon Sep 17 00:00:00 2001 From: mirko Date: Sat, 21 Nov 2020 16:33:50 +0100 Subject: [PATCH 10/22] set master branch to current version of upstream repo --- docs/.DS_Store | Bin 0 -> 6148 bytes src/Term.jl | 2 +- src/expectation_propagation.jl | 4 +-- src/priors.jl | 43 +++++++++------------------------ 4 files changed, 13 insertions(+), 36 deletions(-) create mode 100644 docs/.DS_Store diff --git a/docs/.DS_Store b/docs/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..4e598026ee7d28409daec778ca3d208c340c68e6 GIT binary patch literal 6148 zcmeHK&1%~~5T3Q2X6>}pheC4POQ8o-A{9cR6oi``ato-mhc+aVEfTe|Hdt~^>$>PQ z(l_X%+NQNvw22 z11iBO(Hqt4zTo}2#cNUl-oQK^brsvZyDFX*XK|*hF5mKOyeDR_eXl2Wd~ZH)^Udep+x_Fw_aC#L^Izvm z9lQy66|_CISi^4wR)lZiIL~B$f=pdqS2=VFI0c*nD^S3Gdicf)=5kXw1)Ks8O9AzL z5O9To#mb=mbfD2k0ALGgZHU!Rqv8mQfyK%oMqtcPfriTL5rY{z^1{{y7Au2>PRt%Y zn4Ou~6AII_+f} ymu^n&wE}(&SElMJg9{1*y%nQYZN*z~ZO98100WDaL1bX=M?lNq8mGWY6}Se?=XBNp literal 0 HcmV?d00001 diff --git a/src/Term.jl b/src/Term.jl index dfb048e..7cf6870 100644 --- a/src/Term.jl +++ b/src/Term.jl @@ -19,7 +19,7 @@ mutable struct Term{T <: Real} M::Int end -Term(A,y,β = 1.0) = Term(A,y,0.0,β,0.0,0) +Term(A,y,β = one(eltype(A))) = Term{eltype(A)}(A,y,zero(eltype(A)),eltype(A)(β),zero(eltype(A)),0) function (t::Term)(v::Vector) return v⋅(t.A*v-2*t.y) + t.c diff --git a/src/expectation_propagation.jl b/src/expectation_propagation.jl index 058be87..9e3aacf 100644 --- a/src/expectation_propagation.jl +++ b/src/expectation_propagation.jl @@ -113,7 +113,6 @@ function expectation_propagation(H::AbstractVector{Term{T}}, P0::AbstractVector{ sum!(A,y,H) Δμ, Δs, Δav, Δva = 0.0, 0.0, 0.0, 0.0 A .+= Diagonal(1 ./ b[1:Nx]) .+ Fp * Diagonal(1 ./ b[Nx+1:end]) * F - @assert isposdef(A) Σ .= inverter(A) ax, bx, ay, by = (@view a[1:Nx]), (@view b[1:Nx]), (@view a[Nx+1:end]), (@view b[Nx+1:end]) v .= Σ * (y .+ ax ./ bx .+ (Fp * ((ay-d) ./ by))) @@ -127,7 +126,6 @@ function expectation_propagation(H::AbstractVector{Term{T}}, P0::AbstractVector{ vv = dot(x, v) + d[i-Nx] end - @warn "iter $iter, variable $i: ss=$ss, b[$i]=$(b[i])" if ss < b[i] Δs = max(Δs, update_err!(s, i, clamp(1/(1/ss - 1/b[i]), minvar, maxvar))) Δμ = max(Δμ, update_err!(μ, i, s[i] * (vv/ss - a[i]/b[i]))) @@ -155,7 +153,7 @@ function expectation_propagation(H::AbstractVector{Term{T}}, P0::AbstractVector{ for i in 1:length(H) updateβ(H[i], av[1:Nx]) end - callback(av,Δav,va,Δva,epsconv,maxiter,H,P0) + callback(av,Δav,epsconv,maxiter,H,P0) if Δav < epsconv && norm(F*av[1:Nx]+d-av[Nx+1:end]) < 1e-4 return EPOut(state, :converged) end diff --git a/src/priors.jl b/src/priors.jl index 56d2108..83d35cf 100644 --- a/src/priors.jl +++ b/src/priors.jl @@ -88,6 +88,7 @@ mutable struct SpikeSlabPrior{T<:Real} <: Prior δλ::T end +SpikeSlabPrior(ρ,λ) = SpikeSlabPrior(ρ,λ,0.0,0.0); """ ``p = \\frac1{(ℓ+1)((1/ρ-1) e^{-\\frac12 (μ/σ)^2 (2-\\frac1{1+ℓ})}\\sqrt{1+\\frac1{ℓ}}+1)}`` @@ -179,15 +180,23 @@ end This is a fake Prior that can be used to fix experimental moments Parameters: μ, v (variance, not std) """ -struct PosteriorPrior{T<:Real} <: Prior +struct Posterior2Prior{T<:Real} <: Prior μ::T v::T end -function moments(p0::PosteriorPrior, μ, σ) +function moments(p0::Posterior2Prior, μ, σ) return p0.μ,p0.v end +struct Posterior1Prior{T<:Real} <: Prior + μ::T +end + +function moments(p0::Posterior1Prior, μ, σ) + return p0.μ,σ^2 +end + struct QuadraturePrior{T<:Real} <: Prior f @@ -289,33 +298,3 @@ function moments(::ThetaPrior,μ,σ) return av,var end -""" -Bernoulli-Bernoulli RBM -""" -struct RBM_Bias_Factor{T<:Real} <: Prior - g::T -end - -function moments(p0::RBM_Bias_Factor,μ,σ) - arg=-p0.g+(1.0-2.0*μ)/(2.0*σ^2.0) - av=1.0/(1.0+exp(arg)) - va=0.5/(1.0+cosh(arg)) - return av,va -end - -""" -Gaussian RBM units -""" -struct RBM_Gaussian_Factor{T<:Real} <: Prior - γ::T - θ::T -end - -function moments(p0::RBM_Gaussian_Factor,μ,σ) - av = (μ+p0.θ*σ^2.0)/(1.0+p0.γ*σ^2.0) - secmom = ((μ+p0.θ*σ^2.0)^2.0+σ^2.0*(1.0+p0.γ*σ^2.0))/(1+p0.γ*σ^2.0)^2.0 - #secmom = (μ^2.0+σ^2.0+2.0*p0.θ*μ*σ^2.0+(p0.γ+p0.θ^2.0)*σ^2.0)/(1+p0.γ*σ^2.0)^2.0 - #va = clamp(secmom-av^2.0,1e-50,1e50) - va=1.0/(1.0/σ^2.0+p0.γ) - return av,va -end From 54e9c7a157eec02f178dafd0f747864dd3d24627 Mon Sep 17 00:00:00 2001 From: mirko Date: Tue, 16 Mar 2021 17:32:49 +0100 Subject: [PATCH 11/22] Added theta mixture prior --- src/priors.jl | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/priors.jl b/src/priors.jl index 9b49632..1d7ad2c 100644 --- a/src/priors.jl +++ b/src/priors.jl @@ -297,3 +297,35 @@ function moments(::ThetaPrior,μ,σ) var=σ^2*(1-α*pdf_cf(α)-pdf_cf(α)^2) return av,var end + +""" +A mixture of theta priors: p_0(x)=η*Θ(x)+(1-η)*Θ(-x) +""" +mutable struct ThetaMixturePrior{T<:Real} <: Prior + η::T + δη::T +end + +function theta_mixt_factor(x,η) + f=exp(-0.5*x^2.0)/(η*erfc(-sqrt(0.5)*x)+(1.0-η)*erfc(sqrt(0.5)*x)) + return f +end + +function moments(p0::ThetaMixturePrior,μ,σ) + η=p0.η + α=μ/σ + f=theta_mixt_factor(α,η) + χ=sqrt(2.0/π)*(2.0*η-1.0)*f + av=μ+σ*χ + va=σ^2.0*(1-χ^2.0)-μ*σ*χ + return av,va +end + +function gradient(p0::ThetaMixturePrior,μ,σ) + η=p0.η + x=μ/σ/sqrt(2) + num=2*erf(x) + den=η*erfc(-x)+(1-η)*erfc(x) + p0.η+=p0.δη*num/den + p0.η=clamp(p0.η,0,1) +end From 246c6a1170857192f74d5cb6ee47c099551b9c79 Mon Sep 17 00:00:00 2001 From: mirko Date: Thu, 4 Apr 2019 23:00:29 +0200 Subject: [PATCH 12/22] displaying my own documentation --- docs/make.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/make.jl b/docs/make.jl index 156d905..c455018 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,6 +6,6 @@ makedocs(sitename = "GaussianEP", doctest = true) deploydocs( branch = "gh-pages", - repo = "github.com/abraunst/GaussianEP.git", + repo = "github.com/mpieropan/GaussianEP.git", versions = ["stable" => "v^"] ) From a963adc227dd11e6b3d192e1334dbff09507bc4f Mon Sep 17 00:00:00 2001 From: mirko Date: Fri, 12 Apr 2019 13:41:27 +0200 Subject: [PATCH 13/22] restored github.com/abraunst/GaussianEP.git in deploydocs repo --- docs/make.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/make.jl b/docs/make.jl index c455018..156d905 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,6 +6,6 @@ makedocs(sitename = "GaussianEP", doctest = true) deploydocs( branch = "gh-pages", - repo = "github.com/mpieropan/GaussianEP.git", + repo = "github.com/abraunst/GaussianEP.git", versions = ["stable" => "v^"] ) From 2a5e45f07aba8a77c1b9f45313f556126358a22d Mon Sep 17 00:00:00 2001 From: mirko Date: Mon, 17 Aug 2020 22:39:42 +0200 Subject: [PATCH 14/22] added Gaussian and Bernoulli RBM units --- src/priors.jl | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/priors.jl b/src/priors.jl index 83d35cf..590c330 100644 --- a/src/priors.jl +++ b/src/priors.jl @@ -298,3 +298,33 @@ function moments(::ThetaPrior,μ,σ) return av,var end +""" +Bernoulli-Bernoulli RBM +""" +struct RBM_Bias_Factor{T<:Real} <: Prior + g::T +end + +function moments(p0::RBM_Bias_Factor,μ,σ) + arg=-p0.g+(1.0-2.0*μ)/(2.0*σ^2.0) + av=1.0/(1.0+exp(arg)) + va=0.5/(1.0+cosh(arg)) + return av,va +end + +""" +Gaussian RBM units +""" +struct RBM_Gaussian_Factor{T<:Real} <: Prior + γ::T + θ::T +end + +function moments(p0::RBM_Gaussian_Factor,μ,σ) + av = (μ+p0.θ*σ^2.0)/(1.0+p0.γ*σ^2.0) + secmom = ((μ+p0.θ*σ^2.0)^2.0+σ^2.0*(1.0+p0.γ*σ^2.0))/(1+p0.γ*σ^2.0)^2.0 + #secmom = (μ^2.0+σ^2.0+2.0*p0.θ*μ*σ^2.0+(p0.γ+p0.θ^2.0)*σ^2.0)/(1+p0.γ*σ^2.0)^2.0 + #va = clamp(secmom-av^2.0,1e-50,1e50) + va=1.0/(1.0/σ^2.0+p0.γ) + return av,va +end From d0a699ccebfe138da38a709d03888f18032ee849 Mon Sep 17 00:00:00 2001 From: mirko Date: Fri, 25 Sep 2020 22:29:07 +0200 Subject: [PATCH 15/22] Added posdef assertion for matrix A --- src/expectation_propagation.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/expectation_propagation.jl b/src/expectation_propagation.jl index d0efa4e..6e18006 100644 --- a/src/expectation_propagation.jl +++ b/src/expectation_propagation.jl @@ -113,6 +113,7 @@ function expectation_propagation(H::AbstractVector{Term{T}}, P0::AbstractVector{ sum!(A,y,H) Δμ, Δs, Δav, Δva = 0.0, 0.0, 0.0, 0.0 A .+= Diagonal(1 ./ b[1:Nx]) .+ Fp * Diagonal(1 ./ b[Nx+1:end]) * F + @assert isposdef(A) Σ .= inverter(A) ax, bx, ay, by = (@view a[1:Nx]), (@view b[1:Nx]), (@view a[Nx+1:end]), (@view b[Nx+1:end]) v .= Σ * (y .+ ax ./ bx .+ (Fp * ((ay-d) ./ by))) From 20d54ee6f9bd5106730a08f5b0a521f440a22bf6 Mon Sep 17 00:00:00 2001 From: Mirko Pieropan Date: Fri, 20 Nov 2020 13:49:51 +0100 Subject: [PATCH 16/22] added ThetaMixturePrior --- src/priors.jl | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/src/priors.jl b/src/priors.jl index 590c330..93aae16 100644 --- a/src/priors.jl +++ b/src/priors.jl @@ -298,6 +298,77 @@ function moments(::ThetaPrior,μ,σ) return av,var end +""" +A mixture of theta priors: p_0(x)=η*Θ(x)+(1-η)*Θ(-x) +""" +mutable struct ThetaMixturePrior{T<:Real} <: Prior + η::T + δη::T +end + +function theta_mixt_factor(x,η) + if abs(x)<=Inf + f=exp(-0.5*x^2.0)/(η*erfc(-sqrt(0.5)*x)+(1.0-η)*erfc(sqrt(0.5)*x)) + else + println("In theta_mixt_factor sono *qui*!") + if η!=0.0 + f=0.5*exp(-0.5*x^2.0)/η + else + f=sqrt(0.5*π)/(1.0/x-1.0/x^3.0+3.0/x^5.0) + end + end +end + +############## +mutable struct ThetaMixturePrior2{T<:Real} <: Prior + η::T + δη::T + thr::T +end + +function theta_mixt_factor2(x,η,thr) + if abs(x)<=thr + f=exp(-0.5*x^2.0)/(η*erfc(-sqrt(0.5)*x)+(1.0-η)*erfc(sqrt(0.5)*x)) + else + println("In theta_mixt_factor sono *qui*!") + if η!=0.0 + f=0.5*exp(-0.5*x^2.0)/η + else + f=sqrt(0.5*π)/(1.0/x-1.0/x^3.0+3.0/x^5.0) + end + end +end + +function moments(p0::ThetaMixturePrior2,μ,σ) + η=p0.η + α=μ/σ + f=theta_mixt_factor2(α,η,p0.thr) + χ=sqrt(2.0/π)*(2.0*η-1.0)*f + av=μ+σ*χ + va=σ^2.0*(1-χ^2.0)-μ*σ*χ + return av,va +end +############### + +function moments(p0::ThetaMixturePrior,μ,σ) + η=p0.η + α=μ/σ + f=theta_mixt_factor(α,η) + χ=sqrt(2.0/π)*(2.0*η-1.0)*f + av=μ+σ*χ + va=σ^2.0*(1-χ^2.0)-μ*σ*χ + return av,va +end + +function gradient(p0::ThetaMixturePrior,μ,σ,∂𝐹::Union{FreeEnGrad,Nothing}) + η=p0.η + x=μ/σ/sqrt(2) + num=2*erf(x) + den=η*erfc(-x)+(1-η)*erfc(x) + p0.η+=p0.δη*num/den + p0.η=clamp(p0.η,0,1) +end + """ Bernoulli-Bernoulli RBM """ From abd819921058e8a2eadc0bfb29d0c9aed9bfe66c Mon Sep 17 00:00:00 2001 From: mirko Date: Sat, 21 Nov 2020 16:33:50 +0100 Subject: [PATCH 17/22] set master branch to current version of upstream repo --- docs/.DS_Store | Bin 0 -> 6148 bytes src/expectation_propagation.jl | 1 - src/priors.jl | 102 --------------------------------- 3 files changed, 103 deletions(-) create mode 100644 docs/.DS_Store diff --git a/docs/.DS_Store b/docs/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..4e598026ee7d28409daec778ca3d208c340c68e6 GIT binary patch literal 6148 zcmeHK&1%~~5T3Q2X6>}pheC4POQ8o-A{9cR6oi``ato-mhc+aVEfTe|Hdt~^>$>PQ z(l_X%+NQNvw22 z11iBO(Hqt4zTo}2#cNUl-oQK^brsvZyDFX*XK|*hF5mKOyeDR_eXl2Wd~ZH)^Udep+x_Fw_aC#L^Izvm z9lQy66|_CISi^4wR)lZiIL~B$f=pdqS2=VFI0c*nD^S3Gdicf)=5kXw1)Ks8O9AzL z5O9To#mb=mbfD2k0ALGgZHU!Rqv8mQfyK%oMqtcPfriTL5rY{z^1{{y7Au2>PRt%Y zn4Ou~6AII_+f} ymu^n&wE}(&SElMJg9{1*y%nQYZN*z~ZO98100WDaL1bX=M?lNq8mGWY6}Se?=XBNp literal 0 HcmV?d00001 diff --git a/src/expectation_propagation.jl b/src/expectation_propagation.jl index 6e18006..d0efa4e 100644 --- a/src/expectation_propagation.jl +++ b/src/expectation_propagation.jl @@ -113,7 +113,6 @@ function expectation_propagation(H::AbstractVector{Term{T}}, P0::AbstractVector{ sum!(A,y,H) Δμ, Δs, Δav, Δva = 0.0, 0.0, 0.0, 0.0 A .+= Diagonal(1 ./ b[1:Nx]) .+ Fp * Diagonal(1 ./ b[Nx+1:end]) * F - @assert isposdef(A) Σ .= inverter(A) ax, bx, ay, by = (@view a[1:Nx]), (@view b[1:Nx]), (@view a[Nx+1:end]), (@view b[Nx+1:end]) v .= Σ * (y .+ ax ./ bx .+ (Fp * ((ay-d) ./ by))) diff --git a/src/priors.jl b/src/priors.jl index 93aae16..9b49632 100644 --- a/src/priors.jl +++ b/src/priors.jl @@ -297,105 +297,3 @@ function moments(::ThetaPrior,μ,σ) var=σ^2*(1-α*pdf_cf(α)-pdf_cf(α)^2) return av,var end - -""" -A mixture of theta priors: p_0(x)=η*Θ(x)+(1-η)*Θ(-x) -""" -mutable struct ThetaMixturePrior{T<:Real} <: Prior - η::T - δη::T -end - -function theta_mixt_factor(x,η) - if abs(x)<=Inf - f=exp(-0.5*x^2.0)/(η*erfc(-sqrt(0.5)*x)+(1.0-η)*erfc(sqrt(0.5)*x)) - else - println("In theta_mixt_factor sono *qui*!") - if η!=0.0 - f=0.5*exp(-0.5*x^2.0)/η - else - f=sqrt(0.5*π)/(1.0/x-1.0/x^3.0+3.0/x^5.0) - end - end -end - -############## -mutable struct ThetaMixturePrior2{T<:Real} <: Prior - η::T - δη::T - thr::T -end - -function theta_mixt_factor2(x,η,thr) - if abs(x)<=thr - f=exp(-0.5*x^2.0)/(η*erfc(-sqrt(0.5)*x)+(1.0-η)*erfc(sqrt(0.5)*x)) - else - println("In theta_mixt_factor sono *qui*!") - if η!=0.0 - f=0.5*exp(-0.5*x^2.0)/η - else - f=sqrt(0.5*π)/(1.0/x-1.0/x^3.0+3.0/x^5.0) - end - end -end - -function moments(p0::ThetaMixturePrior2,μ,σ) - η=p0.η - α=μ/σ - f=theta_mixt_factor2(α,η,p0.thr) - χ=sqrt(2.0/π)*(2.0*η-1.0)*f - av=μ+σ*χ - va=σ^2.0*(1-χ^2.0)-μ*σ*χ - return av,va -end -############### - -function moments(p0::ThetaMixturePrior,μ,σ) - η=p0.η - α=μ/σ - f=theta_mixt_factor(α,η) - χ=sqrt(2.0/π)*(2.0*η-1.0)*f - av=μ+σ*χ - va=σ^2.0*(1-χ^2.0)-μ*σ*χ - return av,va -end - -function gradient(p0::ThetaMixturePrior,μ,σ,∂𝐹::Union{FreeEnGrad,Nothing}) - η=p0.η - x=μ/σ/sqrt(2) - num=2*erf(x) - den=η*erfc(-x)+(1-η)*erfc(x) - p0.η+=p0.δη*num/den - p0.η=clamp(p0.η,0,1) -end - -""" -Bernoulli-Bernoulli RBM -""" -struct RBM_Bias_Factor{T<:Real} <: Prior - g::T -end - -function moments(p0::RBM_Bias_Factor,μ,σ) - arg=-p0.g+(1.0-2.0*μ)/(2.0*σ^2.0) - av=1.0/(1.0+exp(arg)) - va=0.5/(1.0+cosh(arg)) - return av,va -end - -""" -Gaussian RBM units -""" -struct RBM_Gaussian_Factor{T<:Real} <: Prior - γ::T - θ::T -end - -function moments(p0::RBM_Gaussian_Factor,μ,σ) - av = (μ+p0.θ*σ^2.0)/(1.0+p0.γ*σ^2.0) - secmom = ((μ+p0.θ*σ^2.0)^2.0+σ^2.0*(1.0+p0.γ*σ^2.0))/(1+p0.γ*σ^2.0)^2.0 - #secmom = (μ^2.0+σ^2.0+2.0*p0.θ*μ*σ^2.0+(p0.γ+p0.θ^2.0)*σ^2.0)/(1+p0.γ*σ^2.0)^2.0 - #va = clamp(secmom-av^2.0,1e-50,1e50) - va=1.0/(1.0/σ^2.0+p0.γ) - return av,va -end From d52a5350d1847f6f1e68eccbc4f49e9e1dcd1e26 Mon Sep 17 00:00:00 2001 From: mirko Date: Tue, 16 Mar 2021 17:32:49 +0100 Subject: [PATCH 18/22] Added theta mixture prior --- src/priors.jl | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/priors.jl b/src/priors.jl index 9b49632..1d7ad2c 100644 --- a/src/priors.jl +++ b/src/priors.jl @@ -297,3 +297,35 @@ function moments(::ThetaPrior,μ,σ) var=σ^2*(1-α*pdf_cf(α)-pdf_cf(α)^2) return av,var end + +""" +A mixture of theta priors: p_0(x)=η*Θ(x)+(1-η)*Θ(-x) +""" +mutable struct ThetaMixturePrior{T<:Real} <: Prior + η::T + δη::T +end + +function theta_mixt_factor(x,η) + f=exp(-0.5*x^2.0)/(η*erfc(-sqrt(0.5)*x)+(1.0-η)*erfc(sqrt(0.5)*x)) + return f +end + +function moments(p0::ThetaMixturePrior,μ,σ) + η=p0.η + α=μ/σ + f=theta_mixt_factor(α,η) + χ=sqrt(2.0/π)*(2.0*η-1.0)*f + av=μ+σ*χ + va=σ^2.0*(1-χ^2.0)-μ*σ*χ + return av,va +end + +function gradient(p0::ThetaMixturePrior,μ,σ) + η=p0.η + x=μ/σ/sqrt(2) + num=2*erf(x) + den=η*erfc(-x)+(1-η)*erfc(x) + p0.η+=p0.δη*num/den + p0.η=clamp(p0.η,0,1) +end From 6918f791bbef0557206929b474a0df595dc9f599 Mon Sep 17 00:00:00 2001 From: mirko Date: Tue, 16 Mar 2021 22:22:06 +0100 Subject: [PATCH 19/22] same src as upstream --- src/GaussianEP.jl | 0 src/ProgressReporter.jl | 0 src/Term.jl | 0 src/expectation_propagation.jl | 0 src/priors.jl | 31 ------------------------------- 5 files changed, 31 deletions(-) mode change 100644 => 100755 src/GaussianEP.jl mode change 100644 => 100755 src/ProgressReporter.jl mode change 100644 => 100755 src/Term.jl mode change 100644 => 100755 src/expectation_propagation.jl mode change 100644 => 100755 src/priors.jl diff --git a/src/GaussianEP.jl b/src/GaussianEP.jl old mode 100644 new mode 100755 diff --git a/src/ProgressReporter.jl b/src/ProgressReporter.jl old mode 100644 new mode 100755 diff --git a/src/Term.jl b/src/Term.jl old mode 100644 new mode 100755 diff --git a/src/expectation_propagation.jl b/src/expectation_propagation.jl old mode 100644 new mode 100755 diff --git a/src/priors.jl b/src/priors.jl old mode 100644 new mode 100755 index 1d7ad2c..83d35cf --- a/src/priors.jl +++ b/src/priors.jl @@ -298,34 +298,3 @@ function moments(::ThetaPrior,μ,σ) return av,var end -""" -A mixture of theta priors: p_0(x)=η*Θ(x)+(1-η)*Θ(-x) -""" -mutable struct ThetaMixturePrior{T<:Real} <: Prior - η::T - δη::T -end - -function theta_mixt_factor(x,η) - f=exp(-0.5*x^2.0)/(η*erfc(-sqrt(0.5)*x)+(1.0-η)*erfc(sqrt(0.5)*x)) - return f -end - -function moments(p0::ThetaMixturePrior,μ,σ) - η=p0.η - α=μ/σ - f=theta_mixt_factor(α,η) - χ=sqrt(2.0/π)*(2.0*η-1.0)*f - av=μ+σ*χ - va=σ^2.0*(1-χ^2.0)-μ*σ*χ - return av,va -end - -function gradient(p0::ThetaMixturePrior,μ,σ) - η=p0.η - x=μ/σ/sqrt(2) - num=2*erf(x) - den=η*erfc(-x)+(1-η)*erfc(x) - p0.η+=p0.δη*num/den - p0.η=clamp(p0.η,0,1) -end From 189b2286cca98d84f2c59b255a44d52c81aa3fe1 Mon Sep 17 00:00:00 2001 From: mirko Date: Tue, 16 Mar 2021 22:30:02 +0100 Subject: [PATCH 20/22] just one line --- src/priors.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/priors.jl b/src/priors.jl index 83d35cf..9b49632 100755 --- a/src/priors.jl +++ b/src/priors.jl @@ -297,4 +297,3 @@ function moments(::ThetaPrior,μ,σ) var=σ^2*(1-α*pdf_cf(α)-pdf_cf(α)^2) return av,var end - From c3cd572f83c162e7effe383e6880aa7a6eadaceb Mon Sep 17 00:00:00 2001 From: mirko Date: Tue, 16 Mar 2021 22:34:27 +0100 Subject: [PATCH 21/22] Added theta mixture prior to priors.jl --- src/priors.jl | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/priors.jl b/src/priors.jl index 9b49632..1d7ad2c 100755 --- a/src/priors.jl +++ b/src/priors.jl @@ -297,3 +297,35 @@ function moments(::ThetaPrior,μ,σ) var=σ^2*(1-α*pdf_cf(α)-pdf_cf(α)^2) return av,var end + +""" +A mixture of theta priors: p_0(x)=η*Θ(x)+(1-η)*Θ(-x) +""" +mutable struct ThetaMixturePrior{T<:Real} <: Prior + η::T + δη::T +end + +function theta_mixt_factor(x,η) + f=exp(-0.5*x^2.0)/(η*erfc(-sqrt(0.5)*x)+(1.0-η)*erfc(sqrt(0.5)*x)) + return f +end + +function moments(p0::ThetaMixturePrior,μ,σ) + η=p0.η + α=μ/σ + f=theta_mixt_factor(α,η) + χ=sqrt(2.0/π)*(2.0*η-1.0)*f + av=μ+σ*χ + va=σ^2.0*(1-χ^2.0)-μ*σ*χ + return av,va +end + +function gradient(p0::ThetaMixturePrior,μ,σ) + η=p0.η + x=μ/σ/sqrt(2) + num=2*erf(x) + den=η*erfc(-x)+(1-η)*erfc(x) + p0.η+=p0.δη*num/den + p0.η=clamp(p0.η,0,1) +end From 98b16be3160488620ff9a3265eb546687f8ec325 Mon Sep 17 00:00:00 2001 From: mirko Date: Wed, 17 Mar 2021 16:44:29 +0100 Subject: [PATCH 22/22] changed params of callback function in expectation_propagation --- src/expectation_propagation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/expectation_propagation.jl b/src/expectation_propagation.jl index d0efa4e..1fa3063 100755 --- a/src/expectation_propagation.jl +++ b/src/expectation_propagation.jl @@ -153,7 +153,7 @@ function expectation_propagation(H::AbstractVector{Term{T}}, P0::AbstractVector{ for i in 1:length(H) updateβ(H[i], av[1:Nx]) end - ret = callback(av,Δav,epsconv,maxiter,H,P0) + ret = callback(iter,state,Δav,Δva,epsconv,maxiter,H,P0) if ret === true || (Δav < epsconv && norm(F*av[1:Nx]+d-av[Nx+1:end]) < 1e-4) return EPOut(state, :converged) end