new syntax

Author: matthieugomez

.travis.yml

@@ -2,11 +2,14 @@ language: julia
 julia:
 - 1.1
 - nightly
+script:
+-  if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
+- julia --project -e 'using Pkg; Pkg.add(PackageSpec(name ="FixedEffects", rev="master")) ; Pkg.add(PackageSpec(name ="FixedEffectModels", rev="master"))  ; Pkg.build(); Pkg.test(; coverage=true)';
 matrix:
   allow_failures:
     - julia: nightly
 after_success:
 - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())'
 notifications:
   on_success: never 
-  on_failure: change
+  on_failure: change

Project.toml

@@ -1,6 +1,6 @@
 name = "InteractiveFixedEffectModels"
 uuid = "80307280-efb2-5c5d-af8b-a9c15821677b"
-version = "0.4.3"
+version = "0.5.0"
 
 [deps]
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
@@ -20,7 +20,7 @@ Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 
 [compat]
 DataFrames = "≥ 0.19.1"
-FixedEffectModels = "≥ 0.8.0"
+FixedEffectModels = "≥ 0.9.0"
 FixedEffects = "≥ 0.3.0"
 LeastSquaresOptim = "≥ 0.7.0"
 StatsBase = "≥ 0.22.0"

README.md

@@ -17,9 +17,7 @@ To estimate an interactive fixed effect model, one needs to specify a formula, a
 ```julia
 using DataFrames, RDatasets, InteractiveFixedEffectModels
 df = dataset("plm", "Cigar")
-df.pState =  categorical(df.State)
-df.pYear =  categorical(df.Year)
-regife(df, @model(Sales ~ Price, ife = (pState + pYear, 2), fe = pState), save = true)
+regife(df, @model(Sales ~ Price + fe(State) + ife(State, Year, 2)), save = true)
 #                      Linear Factor Model                      
 #================================================================
 #Number of obs:             1380  Degree of freedom:          199
@@ -39,45 +37,37 @@ regife(df, @model(Sales ~ Price, ife = (pState + pYear, 2), fe = pState), save =
 		df = dataset("plm", "Cigar")
 		```
 
-		When the only regressor is `0`, `fit` fits a factor model
+		When the only regressor is `0`, `regife` fits a factor model
 		```julia
 		Sales ~ 0
 		```
 
-		Otherwise, `fit` fits a linear model with interactive fixed effects (Bai (2009))
+		Otherwise, `regife` fits a linear model with interactive fixed effects (Bai (2009))
 		```julia
 		Sales ~ Price + Year
 		```
-	- Interactive fixed effects are indicated with the keyword argument `ife`. Variables must be of type `PooledDataVector`. The rank is the number of components to use. facFor instance, for a factor model with id variable `State`, time variable `Year`, and rank `r` equal to 2:
+	
+		Interactive fixed effects are indicated with the function  `ife`. For instance, to specify a factor model with id variable `State`, time variable `Year`, and rank 2, use `ife(State, Year, 2)`.
 
-		```julia
-		df.pState =  categorical(df.State)
-		df.pYear =  categorical(df.Year)
-		ife = (pState + pYear, 2)
-		```
+		High-dimensional Fixed effects can be used, as in `fe(State)` but only for the variables specified in the factor model. See [FixedEffectModels.jl](https://github.com/matthieugomez/FixedEffectModels.jl) for more information
 
-	- Fixed effects are indicated with the keyword argument `fe`. Use only the variables specified in the factor model. See [FixedEffectModels.jl](https://github.com/matthieugomez/FixedEffectModels.jl) for more information
-
-		```julia
-		fe = pState
-		fe = pYear
-		fe = pState + pYear
-		```
 
 	- Standard errors are indicated with the keyword argument `vcov`
 		```julia
 		vcov = robust()
-		vcov = cluster(StatePooled)
-		vcov = cluster(StatePooled, YearPooled)
+		vcov = cluster(State)
+		vcov = cluster(State, Year)
 		```
 
-	- weights are indicated with the keyword argument `weights`
-		```julia
-		weights = Pop
-		```
+- The option `weights` can add weights
+
+	```julia
+	weights = :Pop
+	```
+	
 - The option `method` can be used to choose between two algorithms:
-	- `levenberg_marquardt`
-	- `dogleg` 
+	- `:levenberg_marquardt`
+	- `:dogleg` 
 - The option `save = true` saves a new dataframe storing residuals, factors, loadings and the eventual fixed effects. Importantly, the returned dataframe is aligned with the initial dataframe (rows not used in the estimation are simply filled with NA).
 
 
@@ -95,9 +85,7 @@ Yes. Factor models are a particular case of interactive fixed effect models. Sim
 ```julia
 using DataFrames, RDatasets, InteractiveFixedEffectModels
 df = dataset("plm", "Cigar")
-df.pState =  categorical(df.State)
-df.pYear =  categorical(df.Year)
-regife(df, @model(Sales ~ 0, ife = (pState + pYear, 2), fe = pState), save = true)
+regife(df, @model(Sales ~ 0 + ife(State, Year, 2) + fe(State)), save = true)
 ```
 Compared to the usual SVD method, the package estimates models with multiple (or missing) observations per id x time.
 

src/InteractiveFixedEffectModels.jl

@@ -20,19 +20,22 @@ import LeastSquaresOptim
 using FillArrays
 using Distributions
 using Reexport
-import StatsModels: @formula,  FormulaTerm, Term, InteractionTerm, ConstantTerm, MatrixTerm, AbstractTerm, coefnames, columntable, missing_omit, termvars, schema, apply_schema, modelmatrix, response, terms
+import StatsModels: @formula,  FormulaTerm, Term, InteractionTerm, ConstantTerm, MatrixTerm, AbstractTerm, coefnames, columntable, missing_omit, termvars, schema, apply_schema, modelmatrix, response, terms, FunctionTerm
+using FixedEffects
 @reexport using FixedEffectModels
 
-
+if !isdefined(FixedEffects, :AbstractFixedEffectSolver)
+	AbstractFixedEffectSolver{T} = AbstractFixedEffectMatrix{T}
+end
 
 ##############################################################################
 ##
 ## Exported methods and types 
 ##
 ##############################################################################
 
-export InteractiveFixedEffectFormula,
-InteractiveFixedEffectResult,
+export InteractiveFixedEffectModel, 
+ife,
 regife
 ##############################################################################
 ##
@@ -41,11 +44,9 @@ regife
 ##############################################################################
 include("types.jl")
 
-include("utils/models.jl")
-
 include("methods/gauss_seidel.jl")
 include("methods/ls.jl")
 
-include("regife.jl")
+include("fit.jl")
 
 end

src/regife.jl renamed to src/fit.jl

@@ -4,15 +4,18 @@
 ## Fit is the only exported function
 ##
 ##############################################################################
+function StatsBase.fit(::Type{InteractiveFixedEffectModel}, m::Model, df::AbstractDataFrame; kwargs...)
+    regife(m, df; kwargs...)
+end
 
 
 function regife(df::AbstractDataFrame, m::Model; kwargs...)
     regife(df, m.f; m.dict..., kwargs...)
 end
 
 function regife(df::AbstractDataFrame, f::FormulaTerm;
-             ife::Union{Symbol, Expr, Nothing} = nothing, 
-             fe::Union{Symbol, Expr, Nothing} = nothing, 
+             feformula::Union{Symbol, Expr, Nothing} = nothing,
+             ifeformula::Union{Symbol, Expr, Nothing} = nothing,
              vcov::Union{Symbol, Expr, Nothing} = :(simple()), 
              weights::Union{Symbol, Expr, Nothing} = nothing, 
              subset::Union{Symbol, Expr, Nothing} = nothing, 
@@ -34,33 +37,36 @@ function regife(df::AbstractDataFrame, f::FormulaTerm;
     else 
         vcovformula = VcovFormula(Val{vcov.args[1]}, (vcov.args[i] for i in 2:length(vcov.args))...)
     end
-    m = InteractiveFixedEffectFormula(ife)
 
     if  (ConstantTerm(0) ∉ FixedEffectModels.eachterm(f.rhs)) & (ConstantTerm(1) ∉ FixedEffectModels.eachterm(f.rhs))
-        f = FormulaTerm(f.lhs, tuple(ConstantTerm(1), FixedEffectModels.eachterm(f.rhs)...))
+        formula = FormulaTerm(f.lhs, tuple(ConstantTerm(1), FixedEffectModels.eachterm(f.rhs)...))
     end
+
     formula, formula_endo, formula_iv = FixedEffectModels.decompose_iv(f)
+
+    m, formula = parse_interactivefixedeffect(df, formula)
+    if ifeformula != nothing # remove after depreciation
+        m = OldInteractiveFixedEffectFormula(ifeformula)
+    end
+
     ## parse formula 
     if formula_iv != nothing
         error("partial_out does not support instrumental variables")
     end
-    has_absorb = fe != nothing
     has_weights = (weights != nothing)
 
 
     ## create a dataframe without missing values & negative weightss
     vars = allvars(formula)
-    absorb_vars = allvars(fe)
+    if feformula != nothing # remove after depreciation
+        vars = vcat(vars, allvars(feformula))
+    end
     vcov_vars = allvars(vcovformula)
     factor_vars = vcat(allvars(m.id), allvars(m.time))
-    rem = setdiff(absorb_vars, factor_vars)
-    if length(rem) > 0
-        error("The categorical variable $(rem[1]) appears in @fe but does not appear in @ife. Simply add it in @formula instead")
-    end
-    all_vars = unique(vcat(vars, absorb_vars, factor_vars, vcov_vars))
+    all_vars = unique(vcat(vars, factor_vars, vcov_vars))
     esample = completecases(df[!, all_vars])
     if has_weights
-        esample .&= isnaorneg(df[!, weights])
+        esample .&= FixedEffectModels.isnaorneg(df[!, weights])
         all_vars = unique(vcat(all_vars, weights))
     end
     if subset != nothing
@@ -77,41 +83,43 @@ function regife(df::AbstractDataFrame, f::FormulaTerm;
     vcov_method_data = VcovMethod(df[esample, unique(Symbol.(vcov_vars))], vcovformula)
 
      # Compute weights
-     sqrtw = get_weights(df, esample, weights)
-
+    sqrtw = Ones{Float64}(sum(esample))
+    if has_weights
+        sqrtw = convert(Vector{Float64}, sqrt.(view(df, esample, weights)))
+    end
+    for a in FixedEffectModels.eachterm(formula.rhs)
+       if has_fe(a)
+           isa(a, InteractionTerm) && error("Fixed effects cannot be interacted")
+           Symbol(first(a.args_parsed)) ∉ factor_vars && error("FixedEffect should correspond to id or time dimension of the factor model")
+       end
+    end
+    fes, ids, formula = FixedEffectModels.parse_fixedeffect(df, formula)
+    if feformula != nothing # remove after depreciation
+        feformula = @eval(@formula(0 ~ $(feformula)))
+        fes, ids = FixedEffectModels.oldparse_fixedeffect(df, feformula)
+    end 
+    has_fes = !isempty(fes)
+    has_fes_intercept = false
     ## Compute factors, an array of AbtractFixedEffects
-    if has_absorb
-        feformula = @eval(@formula(nothing ~ $(fe)))
-        fes, ids = FixedEffectModels.parse_fixedeffect(df, feformula)
+    if has_fes
         if any([isa(fe.interaction, Ones) for fe in fes])
                 formula = FormulaTerm(formula.lhs, tuple(ConstantTerm(0), (t for t in FixedEffectModels.eachterm(formula.rhs) if t!= ConstantTerm(1))...))
-                has_absorb_intercept = true
+                has_fes_intercept = true
         end
         fes = FixedEffect[FixedEffectModels._subset(fe, esample) for fe in fes]
-        feM = FixedEffectModels.AbstractFixedEffectMatrix{Float64}(fes, sqrtw, Val{:lsmr})
+        feM = FixedEffectModels.AbstractFixedEffectSolver{Float64}(fes, sqrtw, Val{:lsmr})
     end
 
+
     has_intercept = ConstantTerm(1) ∈ FixedEffectModels.eachterm(formula.rhs)
 
 
     iterations = 0
     converged = false
     # get two dimensions
 
-    if isa(m.id, Symbol)
-        # always factorize
-        id = group(df[esample, m.id])
-    else
-        factorvars, interactionvars = _split(df, allvars(m.id))
-        id = group((df[esample, v] for v in factorvars)...)
-    end
-    if isa(m.time, Symbol)
-        # always factorize
-        time = group(df[esample, m.time])
-    else
-        factorvars, interactionvars = _split(df, allvars(m.time))
-        time = group((df[esample, v] for v in factorvars)...)
-    end
+    id = FixedEffects.group(df[esample, m.id])
+    time = FixedEffects.group(df[esample, m.time])
 
     ##############################################################################
     ##
@@ -144,7 +152,7 @@ function regife(df::AbstractDataFrame, f::FormulaTerm;
 
 
 
-    if has_absorb
+    if has_fes
         FixedEffectModels.solve_residuals!(y, feM)
         FixedEffectModels.solve_residuals!(X, feM)
      end
@@ -191,7 +199,7 @@ function regife(df::AbstractDataFrame, f::FormulaTerm;
             # y ~ x + γ1 x factors + γ2 x loadings
             # if not, this means fit! ended up on a a local minimum. 
             # restart with randomized coefficients, factors, loadings
-            newfeM = FixedEffectModels.AbstractFixedEffectMatrix{Float64}(getfactors(fp, fs), sqrtw, Val{:lsmr})
+            newfeM = FixedEffectModels.AbstractFixedEffectSolver{Float64}(getfactors(fp, fs), sqrtw, Val{:lsmr})
             ym .= ym ./sqrtw
             FixedEffectModels.solve_residuals!(ym, newfeM, tol = tol, maxiter = maxiter)
             ym .= ym .* sqrtw
@@ -236,7 +244,7 @@ function regife(df::AbstractDataFrame, f::FormulaTerm;
         crossxm = cholesky!(Symmetric(Xm' * Xm))
         ## compute the right degree of freedom
         df_absorb_fe = 0
-        if has_absorb 
+        if has_fes 
             for fe in fes
                 df_absorb_fe += length(unique(fe.refs))
             end
@@ -249,13 +257,13 @@ function regife(df::AbstractDataFrame, f::FormulaTerm;
         # compute various r2
         nobs = sum(esample)
         rss = sum(abs2, residualsm)
-        tss = compute_tss(ym, has_intercept || has_absorb_intercept, sqrtw)
-        r2_within = 1 - rss / tss 
+        _tss = FixedEffectModels.tss(ym, has_intercept || has_fes_intercept, sqrtw)
+        r2_within = 1 - rss / _tss 
 
         rss = sum(abs2, residuals)
-        tss = compute_tss(oldy, has_intercept || has_absorb_intercept, sqrtw)
-        r2 = 1 - rss / tss 
-        r2_a = 1 - rss / tss * (nobs - has_intercept) / dof_residual 
+        _tss = FixedEffectModels.tss(oldy, has_intercept || has_fes_intercept, sqrtw)
+        r2 = 1 - rss / _tss 
+        r2_a = 1 - rss / _tss * (nobs - has_intercept) / dof_residual 
     end
 
     ##############################################################################
@@ -277,7 +285,7 @@ function regife(df::AbstractDataFrame, f::FormulaTerm;
         end
 
         # save fixed effects in a dataframe
-        if has_absorb
+        if has_fes
             # residual before demeaning
              oldresiduals = convert(Vector{Float64}, response(formula_schema, subdf))
              oldresiduals .= oldresiduals .* sqrtw