C Please keep this header info with the program. I will maintain it
C so send your comments and suggestions to--
C
C Author: David Zubrow. 8356 Post Rd. Allison Park, Pa 15101-3225
C 412-367-8956
C E-Mail Zubrow@Andrew.cmu.edu
C
C Last Revised: Dec 17, 1987
C All rights reserved. Copyright 1991.
C This program calculates KAPPA according to Fleiss, Psych Bull
C 1971, Vol. 76, No. 5, p 378-382
C To increase or decrease the arrays for the problem edit the following
C DIMENSION statement and the MAX??? vars under the current maximum
C limit comment. ICASE is dimensioned as Cases by Judges. KCASE is
C dimensioned as Cases by Categories. P is dimensioned as Categories.
C RKP is dimensioned as Categories and holds the individual category
C kappas.
C XIJ holds the sum of the frequencies squared. P2 is used for the
C kappa calculation and P3 is used in the calculation of its variance.
C IXJ holds the sum of the frequencies squared for each category. RKPSE
C and RKPZ hold the computed standard error and Z-stats for each
C category.
C
C Compute kappas for all but one judges. Who is dragging it down?
C The following code creates a table whose rows contain the index
C for each judge. Each row excludes the index for one judge. This
C must be disabled if the number of judges is less than 3.
C The routine should also report who was left out.
C
C DIMENSION INEW(5,5)
C INTEGER N
C N=5
C
C DO 40 M=1,N
C ICOL=0
C IF(M.EQ.N) GOTO 43
C DO 41 J=1,N-M
C ICOL=ICOL+1
C INEW(M,ICOL)=J
C 41 CONTINUE
C IF(ICOL.GE.N-1) GOTO 40
C 43 DO 42 J=N-M+2,N
C ICOL=ICOL+1
C INEW(M,ICOL)=J
C 42 CONTINUE
C 40 CONTINUE
C
C DO 50 I=1,N
C PRINT *,(INEW(I,J),J=1,N-1)
C 50 CONTINUE
C STOP
C END
C
DIMENSION ICASE(400,10),KCASE(400,125),P(125),RKP(125),IXJ(125),
1RKPSE(125),RKPZ(125)
INTEGER ICAT,N,JUD,MAXCAT,MAXN,MAXJUD
REAL XIJ,SEKAP,XKAPPA,ZSCORE
CHARACTER*20 DATAIN, DATOUT
C
C CURRENT MAXIMUM LIMITS ON PROBLEM SIZE
MAXCAT=125
MAXN=400
MAXJUD=10
C INPUT SOME PARAMETERS
PRINT 4
4 FORMAT(' ')
PRINT 5
5 FORMAT(' Kappa Calculation Program')
PRINT 4
PRINT 17
17 FORMAT(' Current Problem Limits are')
PRINT 18,MAXCAT
18 FORMAT(I4,' Categories')
PRINT 19,MAXN
19 FORMAT(I4,' Cases')
PRINT 20,MAXJUD
20 FORMAT(I3,' Judges')
PRINT 4
PRINT 30
30 FORMAT(' Enter the input file name: ')
ACCEPT 32,DATAIN
32 FORMAT(A10)
PRINT 4
PRINT 31
31 FORMAT(' Enter the output file name: ')
ACCEPT 33,DATOUT
33 FORMAT(A10)
PRINT 4
PRINT 1
1 FORMAT(' Enter the number of categories: ')
ACCEPT *,ICAT
C CHANGE GOTO 12 TO GOTO 11 IN FOLLOWING LINE AND UNCOMMENT CASES TO REVERT
C TO OLD PROGRAM
IF (ICAT .LE. MAXCAT) GOTO 12
PRINT 14
14 FORMAT(' **** Maximum Number of Categories Exceeded.
1 Program Terminating. ****')
PRINT 4
STOP
C11 PRINT 2
C2 FORMAT(' Enter the number of cases: ')
C ACCEPT *,N
C IF (N .LE. MAXN) GOTO 12
C PRINT 15
C15 FORMAT(' **** Maximum Number of Cases Exceeded.
C 1 Program Terminating. ****')
C PRINT 4
C STOP
12 PRINT 3
3 FORMAT(' Enter the number of judges: ')
ACCEPT *,JUD
IF (JUD .LE. MAXJUD) GOTO 13
PRINT 16
16 FORMAT(' **** Maximum Number of Judges Exceeded.
1 Program Terminating. ****')
PRINT 4
STOP
C MAIN PROGRAM
13 OPEN (UNIT=2,FILE=DATAIN)
OPEN (UNIT=3,FILE=DATOUT)
PRINT 4
PRINT 4
CALL SETUP (ICASE,N,JUD,DATAIN,MAXN)
CALL NEWK (ICASE,KCASE,N,JUD,ICAT,MAXN)
CALL CALC (KCASE,N,JUD,P,XIJ,ICAT,XKAPPA,ZSCORE,SEKAP,RKP,IXJ,
1RKPSE,RKPZ)
CALL PRINT (KCASE,XKAPPA,SEKAP,ZSCORE,P,N,ICAT,RKP,RKPSE,RKPZ,
1DATAIN)
PRINT 4
PRINT 4
END
SUBROUTINE SETUP (ICASE,N,JUD,DATAIN,MAXN)
INTEGER N
DIMENSION ICASE(MAXN,JUD)
CHARACTER*20 DATAIN
N=0
10 N=N+1
READ (2,*,END=21,ERR=999) (ICASE(N,J),J=1,JUD)
GOTO 10
21 CLOSE (2)
N=N-1
PRINT 22,N,DATAIN
22 FORMAT (I5,' cases read from ',A20)
RETURN
999 PRINT 6,N
6 FORMAT(' *** Error on Input Line ',I3'. Program
1 Terminating. ***')
STOP
END
SUBROUTINE NEWK (ICASE,KCASE,N,JUD,ICAT,MAXN)
DIMENSION KCASE(N,ICAT),ICASE(MAXN,JUD)
DO 80 I=1,N
DO 90 K=1,ICAT
KCASE(I,K)=0
90 CONTINUE
80 CONTINUE
DO 100 I=1,N
DO 110 K=1,ICAT
DO 120 J=1,JUD
IF (ICASE(I,J) .EQ. K) KCASE(I,K)=KCASE(I,K)+1
120 CONTINUE
110 CONTINUE
100 CONTINUE
RETURN
END
SUBROUTINE CALC (KCASE,N,JUD,P,XIJ,ICAT,XKAPPA,
1ZSCORE,SEKAP,RKP,IXJ,RKPSE,RKPZ)
DIMENSION KCASE(N,ICAT),P(ICAT),RKP(ICAT),IXJ(ICAT),RKPSE(ICAT),
1RKPZ(ICAT)
REAL P2,P3,VKAPPA
XIJ=0
VKAPPA=0
P2=0
P3=0
DO 200 I=1,N
DO 210 IC=1,ICAT
P(IC)=P(IC)+KCASE(I,IC)
XIJ=KCASE(I,IC)**2+XIJ
IXJ(IC)=KCASE(I,IC)**2+IXJ(IC)
210 CONTINUE
200 CONTINUE
NOBS=N*JUD
N1NOBS=NOBS*(JUD-1)
DO 220 I=1,ICAT
P(I)=P(I)/NOBS
PQ=P(I)*(1-P(I))
P2=P(I)**2+P2
P3=P(I)**3+P3
C
C This computes the kappas and standard errors for each category.
RKP(I)=IXJ(I)-NOBS*P(I)*(1+(JUD-1)*P(I))
RKP(I)=RKP(I)/(N1NOBS*PQ)
RKPSE(I)=((1+2*(JUD-1)*P(I))**2)+2*(JUD-1)*PQ
RKPSE(I)=(RKPSE(I)/(NOBS*((JUD-1)**2)*PQ))**.5
RKPZ(I)=RKP(I)/RKPSE(I)
220 CONTINUE
C Calculation of Kappa
XKAPPA=XIJ-(NOBS*(1+((JUD-1)*P2)))
XKAPPA=XKAPPA/(N1NOBS*(1-P2))
C This is the computation of the variance of kappa
VKAPPA=2.0/N1NOBS
VKAPPA=VKAPPA*(P2-((2*JUD-3)*(P2**2))+2*(JUD-2)*P3)
VKAPPA=VKAPPA/((1-P2)**2)
SEKAP=SQRT(VKAPPA)
ZSCORE=XKAPPA/SEKAP
RETURN
END
SUBROUTINE PRINT (KCASE,XKAPPA,SEKAP,ZSCORE,P,N,ICAT,RKP,RKPSE,
1RKPZ,DATAIN)
DIMENSION KCASE(N,ICAT),P(ICAT),RKP(ICAT),RKPSE(ICAT),RKPZ(ICAT)
CHARACTER*20 DATAIN
WRITE(3,1020) DATAIN
1020 FORMAT(1X,' Results from file ',A20)
WRITE(3,1001)
WRITE(3,1010) XKAPPA
1010 FORMAT(1X,'Kappa = ',F7.4)
WRITE(3,1011) SEKAP
1011 FORMAT(1X,'S.E. of Kappa = ',F7.4)
WRITE(3,1012) ZSCORE
1012 FORMAT(1X,'Z (Test Stat) = ',F7.4)
WRITE(3,1013) N
1013 FORMAT(1X,'N = ',I6)
WRITE (3,1001)
WRITE (3,1002)
1002 FORMAT(' The Kappa, Z-Stat, Std Err, and P of all
1judgments for each Category.')
WRITE (3,1001)
WRITE (3,1015) (I,RKP(I),RKPZ(I),RKPSE(I),P(I),I=1,ICAT)
1015 FORMAT(I5,4F10.4)
WRITE(3,1001)
WRITE(3,1001)
WRITE (3,1000)
1000 FORMAT(' Number of Raters choosing each category by case.')
WRITE (3,1001)
1001 FORMAT(' ')
WRITE (3,1004)
1004 FORMAT(2X,'CASE')
DO 300 I=1,N
WRITE (3,*) I,(KCASE(I,J),J=1,ICAT)
300 CONTINUE
CLOSE (3)
PRINT 32,XKAPPA
32 FORMAT(' Kappa = ',F7.3)
PRINT 9,SEKAP
9 FORMAT(' S.E. of Kappa = ',F7.3)
PRINT 8,ZSCORE
8 FORMAT(' Z (Test Stat) = ',F7.3)
PRINT 7
7 FORMAT(' ')
PRINT 33
33 FORMAT(' The kappas and Z-stats for each category are')
DO 400 I=1,ICAT
PRINT 34,I,RKP(I),RKPZ(I)
34 FORMAT(3X,I4,3X,F6.4,F10.4)
400 CONTINUE
RETURN
END
C
C*** End of Program
C
C This program may distributed free of charge but may not be sold or used for
C commercial purposes without the consent of the author.
Kappa.Hlp
This is the help file for the program KAPPA, by David Zubrow.
Kappa calculates a statistic of agreement among N judges simultaneously. The
computation is taken from Fleiss (Pysch Bulletin, vol 76, no 5 (1971), p
378-382). The program computes an overall Kappa and its corresponding
standard error and Z-statistic as well as these same statistics for each
individual category. Additionally, it reports the proportion of all judgments
in each category and produces a table showing the frequency with which each
category was selected for each case. This table can be useful for identifying
cases containing high agreement and disagreement.
I. Data Format Requirements.
1. Data for input should be in free format form. That is, each datum should
be separated from others by a comma. Only data to be analyzed should be in
the file.
2. The data values for the categories should be consecutive from 1 to N.
This means that category codes should not be 0 or negative. Also, some
inaccuracies might arise if categories that are never selected are included as
potential categories. It is better to recode the data into consecutive
categories with positive frequencies.
II. Problem Size Limitations.
The program displays the maximum limits for a problem. These can easily be
altered by changing the dimensions of the arrays and the corresponding
variables at the beginning of the program. The source code documents these
changes well.
III. Input Requirements at Execution.
The program requires as input from the console the following:
Data File --containing the data to be analyzed
Output File --to store results
N of categories --number of categories judges could use
N of cases --number of cases in the dataset (rows)
N of judges --the number of judges or raters (columns)
These numbers must be exact or the calculations will not be accurate.
Test data taken from the Fleiss article cited above are stored in Kappa.In.
When changes are made to the program, these data can be used to check the
program.
Problems, comments, and suggestions for improvement should be directed to
David Zubrow zubrow@andrew.cmu.edu
Asst Director for Analytic Studies 412-268-2123
University Planning
316 Warner Hall
Carnegie Mellon University
Pittsburgh, Pa 15213-3890
*****
Test Data
****
4 4 4 4 4 4
2 2 2 5 5 5
2 3 3 3 3 5
5 5 5 5 5 5
2 2 2 4 4 4
1 1 3 3 3 3
3 3 3 3 5 5
1 1 3 3 3 4
1 1 4 4 4 4
5 5 5 5 5 5
4 4 4 4 4 1
1 2 4 4 4 4
2 2 2 3 3 3
1 4 4 4 4 4
2 2 4 4 4 5
3 3 3 3 3 5
1 1 1 4 5 5
1 1 1 1 1 2
2 2 4 4 4 4
1 3 3 5 5 5
5 5 5 5 5 5
2 4 4 4 4 4
2 2 4 5 5 5
1 1 4 4 4 4
1 4 4 4 4 5
2 2 2 2 2 4
1 1 1 1 5 5
2 2 4 4 4 4
1 3 3 3 3 3
5 5 5 5 5 5